Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(plancheck): estimate affected rows correctly across engines (#21442)
Danny Xue22be6f5e2d6
diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.goindex f28ba63bd9..125975f2ae 100644--- a/backend/api/v1/release_service_check.go+++ b/backend/api/v1/release_service_check.go@@ -447,18 +447,26 @@ loop:// Get SQL summary report for the statement and target database.// Including affected rows.- summaryReport, err := plancheck.GetSQLSummaryReport(ctx, s.store, s.sheetManager, s.dbFactory, database, statement)+ summaryReport, estimateWarning, err := plancheck.GetSQLSummaryReport(ctx, s.store, s.sheetManager, s.dbFactory, database, statement)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get SQL summary report"))}if summaryReport != nil {checkResult.AffectedRows = summaryReport.AffectedRowscheckResult.RiskLevel = getRiskLevelFromStatementTypes(summaryReport.StatementTypes)- resp.AffectedRows += summaryReport.AffectedRows+ resp.AffectedRows = common.AddRows(resp.AffectedRows, summaryReport.AffectedRows)if checkResult.RiskLevel > resp.RiskLevel {resp.RiskLevel = checkResult.RiskLevel}}+ if estimateWarning != "" {+ checkResult.Advices = append(checkResult.Advices, &v1pb.Advice{+ Status: v1pb.Advice_WARNING,+ Code: code.StatementExplainQueryFailed.Int32(),+ Title: plancheck.AffectedRowsEstimateIncompleteTitle,+ Content: estimateWarning,+ })+ }if common.EngineSupportSQLReview(engine) {adviceStatus, sqlReviewAdvices, err := s.runSQLReviewCheckForFile(ctx, project, originMetadata, finalMetadata, instance, database, statement)if err != nil {diff --git a/backend/common/engine.go b/backend/common/engine.goindex 7ade4471cb..09870374f4 100644--- a/backend/common/engine.go+++ b/backend/common/engine.go@@ -246,7 +246,8 @@ func EngineSupportStatementReport(e storepb.Engine) bool {storepb.Engine_ORACLE,storepb.Engine_MSSQL,storepb.Engine_MARIADB,- storepb.Engine_REDSHIFT:+ storepb.Engine_REDSHIFT,+ storepb.Engine_COCKROACHDB:return truecasestorepb.Engine_ENGINE_UNSPECIFIED,@@ -259,7 +260,6 @@ func EngineSupportStatementReport(e storepb.Engine) bool {storepb.Engine_BIGQUERY,storepb.Engine_STARROCKS,storepb.Engine_HIVE,- storepb.Engine_COCKROACHDB,storepb.Engine_DORIS,storepb.Engine_DYNAMODB,storepb.Engine_ELASTICSEARCH,diff --git a/backend/common/util.go b/backend/common/util.goindex 85a55f6a31..003fdc86a8 100644--- a/backend/common/util.go+++ b/backend/common/util.go@@ -5,6 +5,7 @@ import ("crypto/rand""encoding/base64""fmt"+ "math""math/big""net/url""reflect"@@ -34,6 +35,23 @@ const (MaximumLintExplainSize = 10)+// RoundRows rounds a planner's row estimate to int64, saturating at math.MaxInt64 instead of+// overflowing to a negative count.+func RoundRows(rows float64) int64 {+ if rows >= math.MaxInt64 {+ return math.MaxInt64+ }+ return int64(math.Round(rows))+}++// AddRows adds row counts, saturating at math.MaxInt64 instead of wrapping negative.+func AddRows(a, b int64) int64 {+ if b > 0 && a > math.MaxInt64-b {+ return math.MaxInt64+ }+ return a + b+}+var letters = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")// ProtojsonMarshaler is a global protojson marshaler with DiscardUnknown set to true.diff --git a/backend/common/util_test.go b/backend/common/util_test.goindex 00ee1d1334..2366732da3 100644--- a/backend/common/util_test.go+++ b/backend/common/util_test.go@@ -2,6 +2,7 @@package commonimport (+ "math""testing""unicode/utf8"@@ -449,3 +450,16 @@ func TestSanitizeUTF8MessageNameFields(t *testing.T) {"valid UTF-8 must pass through unchanged")require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))}++func TestRoundRows(t *testing.T) {+ require.Equal(t, int64(3), RoundRows(2.5))+ require.Equal(t, int64(1000), RoundRows(999.6))+ require.Equal(t, int64(math.MaxInt64), RoundRows(math.MaxInt64))+ require.Equal(t, int64(math.MaxInt64), RoundRows(1e30))+}++func TestAddRows(t *testing.T) {+ require.Equal(t, int64(5), AddRows(2, 3))+ require.Equal(t, int64(math.MaxInt64), AddRows(math.MaxInt64, 1))+ require.Equal(t, int64(math.MaxInt64), AddRows(math.MaxInt64-1, math.MaxInt64-1))+}diff --git a/backend/plugin/advisor/contains_ddl_test.go b/backend/plugin/advisor/contains_ddl_test.goindex caf74dc1b5..d639e39f9b 100644--- a/backend/plugin/advisor/contains_ddl_test.go+++ b/backend/plugin/advisor/contains_ddl_test.go@@ -29,6 +29,12 @@ func TestContainsDDL(t *testing.T) {sql: "INSERT INTO t VALUES (1); UPDATE t SET a = 1; DELETE FROM t;",wantDDL: false,},+ {+ name: "PostgreSQL: MERGE is DML",+ engine: storepb.Engine_POSTGRES,+ sql: "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE; UPDATE t SET a = 1;",+ wantDDL: false,+ },{name: "PostgreSQL: DDL only",engine: storepb.Engine_POSTGRES,diff --git a/backend/plugin/advisor/mysql/common.go b/backend/plugin/advisor/mysql/common.godeleted file mode 100644index 935302540f..0000000000--- a/backend/plugin/advisor/mysql/common.go+++ /dev/null@@ -1,12 +0,0 @@-package mysql--import "github.com/pkg/errors"--func getColumnIndex(columns []string, name string) (int, error) {- for i, columnName := range columns {- if columnName == name {- return i, nil- }- }- return -1, errors.Errorf("failed to find column %q", name)-}diff --git a/backend/plugin/advisor/mysql/mysql_rules_test.go b/backend/plugin/advisor/mysql/mysql_rules_test.goindex 1e2cf18f05..a7428fd6b3 100644--- a/backend/plugin/advisor/mysql/mysql_rules_test.go+++ b/backend/plugin/advisor/mysql/mysql_rules_test.go@@ -4,14 +4,16 @@ import ("context""database/sql""database/sql/driver"- "errors""io""strings""testing"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors""github.com/stretchr/testify/require"+ "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/component/sheet"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor"@@ -19,20 +21,24 @@ import ("github.com/bytebase/bytebase/backend/store/model")-var testMySQLAdvisorExplainRows [][]driver.Value+var (+ // testMySQLAdvisorExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries.+ testMySQLAdvisorExplainJSON string-// testMySQLAdvisorExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries; when-// empty, the JSON query returns no rows and the advisor falls back to the tabular rows.-var testMySQLAdvisorExplainJSON string+ // testMySQLAdvisorExplainErr, when set, fails every query, standing in for a+ // statement the server refuses to EXPLAIN.+ testMySQLAdvisorExplainErr error-// testMySQLAdvisorExplainErr, when set, fails every query, standing in for a-// statement the server refuses to EXPLAIN.-var testMySQLAdvisorExplainErr error+ // testMySQLAdvisorSetErr, when set, fails every SET statement.+ testMySQLAdvisorSetErr error-// testMySQLAdvisorQueries records every statement the fake driver is asked to-// run, so a rule that sent raw DML instead of EXPLAIN is caught rather than-// silently succeeding.-var testMySQLAdvisorQueries []string+ // testMySQLAdvisorQueries records every statement the fake driver is asked to+ // run, so a rule that sent raw DML instead of EXPLAIN is caught rather than+ // silently succeeding. testMySQLAdvisorQueryConns records the connection that ran each.+ testMySQLAdvisorQueries []string+ testMySQLAdvisorQueryConns []int+ testMySQLAdvisorConnections int+)func init() {sql.Register("test_mysql_advisor_explain", testMySQLAdvisorExplainDriver{})@@ -41,39 +47,48 @@ func init() {type testMySQLAdvisorExplainDriver struct{}func (testMySQLAdvisorExplainDriver) Open(string) (driver.Conn, error) {- return testMySQLAdvisorExplainConn{}, nil+ testMySQLAdvisorConnections+++ return &testMySQLAdvisorExplainConn{id: testMySQLAdvisorConnections}, nil}-type testMySQLAdvisorExplainConn struct{}+type testMySQLAdvisorExplainConn struct {+ id int+}-func (testMySQLAdvisorExplainConn) Prepare(string) (driver.Stmt, error) {+func (*testMySQLAdvisorExplainConn) Prepare(string) (driver.Stmt, error) {return nil, driver.ErrSkip}-func (testMySQLAdvisorExplainConn) Close() error {+func (*testMySQLAdvisorExplainConn) Close() error {return nil}-func (testMySQLAdvisorExplainConn) Begin() (driver.Tx, error) {+func (*testMySQLAdvisorExplainConn) Begin() (driver.Tx, error) {return testMySQLAdvisorExplainTx{}, nil}-func (testMySQLAdvisorExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+func (c *testMySQLAdvisorExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {+ testMySQLAdvisorQueries = append(testMySQLAdvisorQueries, query)+ testMySQLAdvisorQueryConns = append(testMySQLAdvisorQueryConns, c.id)+ if !strings.HasPrefix(query, "SET ") {+ return nil, errors.Errorf("unexpected exec %q", query)+ }+ if testMySQLAdvisorSetErr != nil {+ return nil, testMySQLAdvisorSetErr+ }+ return driver.ResultNoRows, nil+}++func (c *testMySQLAdvisorExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {testMySQLAdvisorQueries = append(testMySQLAdvisorQueries, query)+ testMySQLAdvisorQueryConns = append(testMySQLAdvisorQueryConns, c.id)if testMySQLAdvisorExplainErr != nil {return nil, testMySQLAdvisorExplainErr}- if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON") {- var rows [][]driver.Value- if testMySQLAdvisorExplainJSON != "" {- rows = [][]driver.Value{{testMySQLAdvisorExplainJSON}}- }- return &testMySQLAdvisorExplainResultRows{columns: []string{"EXPLAIN"}, rows: rows}, nil+ if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON ") {+ return &testMySQLAdvisorExplainResultRows{columns: []string{"EXPLAIN"}, rows: [][]driver.Value{{testMySQLAdvisorExplainJSON}}}, nil}- return &testMySQLAdvisorExplainResultRows{- columns: []string{"id", "select_type", "table", "type", "rows", "filtered"},- rows: testMySQLAdvisorExplainRows,- }, nil+ return &testMySQLAdvisorExplainResultRows{columns: []string{"id", "select_type", "table", "type", "rows", "filtered"}}, nil}type testMySQLAdvisorExplainTx struct{}@@ -187,95 +202,235 @@ func TestMySQLRules(t *testing.T) {}}-func TestMySQLAffectedRowLimitAdvisorUsesJSONPlanTargetNode(t *testing.T) {- // The tabular EXPLAIN's first row is a 100232-row driving-table scan; the JSON- // plan flags the update target whose cumulative estimate is 1144 (BYT-9858).- testMySQLAdvisorExplainJSON = `{"query_block":{"select_id":1,"nested_loop":[- {"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":100232,"rows_produced_per_join":3006,"filtered":"3.00"}},- {"table":{"update":true,"table_name":"o","access_type":"ref","rows_examined_per_scan":3,"rows_produced_per_join":1144,"filtered":"10.00"}}- ]}}`- testMySQLAdvisorExplainRows = [][]driver.Value{- {int64(1), "SIMPLE", "t", "ALL", int64(100232), "3.00"},- {int64(1), "UPDATE", "o", "ref", int64(3), "10.00"},- }- defer func() { testMySQLAdvisorExplainJSON = "" }()-+// openTestMySQLAdvisorDB resets the fake server and opens a pool that keeps no idle+// connections, so statements that do not share a pinned connection land on different ones.+func openTestMySQLAdvisorDB(t *testing.T, plan string) *sql.DB {+ t.Helper()+ testMySQLAdvisorExplainJSON = plan+ testMySQLAdvisorExplainErr = nil+ testMySQLAdvisorSetErr = nil+ testMySQLAdvisorQueries = nil+ testMySQLAdvisorQueryConns = nildb, err := sql.Open("test_mysql_advisor_explain", "")require.NoError(t, err)- defer db.Close()+ db.SetMaxIdleConns(0)+ t.Cleanup(func() { _ = db.Close() })+ return db+}- sm := sheet.NewManager()- adviceList, err := advisor.SQLReviewCheck(context.Background(), sm, "UPDATE o SET c = 1 WHERE c IS NULL AND EXISTS (SELECT 1 FROM t WHERE t.k = o.k);", []*storepb.SQLReviewRule{+func checkTestRowLimitRule(t *testing.T, db *sql.DB, engine storepb.Engine, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, statement string) []*storepb.Advice {+ t.Helper()+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{{- Type: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,- Level: storepb.SQLReviewRule_WARNING,- Engine: storepb.Engine_MYSQL,- Payload: &storepb.SQLReviewRule_NumberPayload{- NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{- Number: 2000,- },- },+ Type: ruleType,+ Level: level,+ Engine: engine,+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: 5}},},}, advisor.Context{- DBType: storepb.Engine_MYSQL,+ DBType: engine,Driver: db,NoAppendBuiltin: true,})require.NoError(t, err)- // 1144 is below the 2000 limit; the legacy first-"rows" heuristic would have- // reported 100232 and produced advice.- require.Empty(t, adviceList)+ return adviceList}-func TestMySQLRowLimitAdvisorsCapExplainEstimateByLimit(t *testing.T) {- testMySQLAdvisorExplainJSON = ""- testMySQLAdvisorExplainRows = [][]driver.Value{- {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},- }- db, err := sql.Open("test_mysql_advisor_explain", "")- require.NoError(t, err)- defer db.Close()+const (+ // The flagged UPDATE target is scanned first; the join filter on s comes after it.+ testMySQLUpdateTargetFirstPlan = `{"query_block":{"select_id":1,"nested_loop":[+ {"table":{"update":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00"}},+ {"table":{"table_name":"s","access_type":"eq_ref","rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00"}}+ ]}}`+ // MariaDB does not flag a multi-table DELETE target.+ testMariaDBDeleteJoinPlan = `{"query_block":{"select_id":1,"nested_loop":[+ {"table":{"table_name":"s","access_type":"ALL","loops":1,"rows":100,"filtered":10}},+ {"table":{"table_name":"big","access_type":"ref","loops":10,"rows":100,"filtered":100}}+ ]}}`+ // The plan of the SELECT that a single-table UPDATE of td is explained as.+ testMySQLSingleTableUpdatePlan = `{"query_block":{"select_id":1,"table":{"table_name":"td","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00"}}}`+ testMySQLInsertSelectPlan = `{"query_block":{"select_id":1,+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}+ }}`+ testMySQLInsertTablePlan = `{"query_block":{"select_id":1,+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00"}}+ }}`+)- sm := sheet.NewManager()+func TestMySQLRowLimitAdvisorsEstimateFromJSONPlan(t *testing.T) {for _, tc := range []struct {name string- statement string+ engine storepb.EngineruleType storepb.SQLReviewRule_Type+ plan string+ setErr error+ statement string+ // explained is the statement the rule explains, when it is not the statement itself.+ explained string+ // wantContent is empty when the statement stays within the limit of 5.+ wantContent string+ wantCode code.Code+ // fallback is set when the JSON plan has no estimate to read, so the rule also runs the tabular EXPLAIN.+ fallback bool}{{- name: "affected row limit",- statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 3;",+ name: "update bounded by the join after the target",+ engine: storepb.Engine_MYSQL,+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,+ plan: testMySQLUpdateTargetFirstPlan,+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",+ wantContent: `"UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;" affected 1133 rows (estimated). The count exceeds 5.`,+ wantCode: code.StatementAffectedRowExceedsLimit,+ },+ {+ name: "MariaDB multi-table delete without explain_json_format_version",+ engine: storepb.Engine_MARIADB,+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,+ plan: testMariaDBDeleteJoinPlan,+ setErr: &mysql.MySQLError{Number: 1193, Message: "Unknown system variable 'explain_json_format_version'"},+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ wantContent: `"DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;" affected 1000 rows (estimated). The count exceeds 5.`,+ wantCode: code.StatementAffectedRowExceedsLimit,+ },+ {+ name: "update capped by LIMIT",+ engine: storepb.Engine_MYSQL,ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,+ plan: testMySQLSingleTableUpdatePlan,+ statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 3;",+ explained: "SELECT 1 FROM td WHERE c = 0 LIMIT 3",+ },+ {+ name: "target the plan does not name",+ engine: storepb.Engine_MYSQL,+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,+ plan: `{"query_block":{"select_id":1,"table":{"table_name":"other","access_type":"ALL","rows_examined_per_scan":1000,"filtered":"100.00"}}}`,+ statement: "UPDATE td SET c = 1;",+ explained: "SELECT 1 FROM td",+ wantContent: `"UPDATE td SET c = 1;" affected 1000 rows (estimated). The count exceeds 5.`,+ wantCode: code.StatementAffectedRowExceedsLimit,},{- name: "insert row limit",- statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 3;",+ name: "plan without an estimate falls back to the tabular plan",+ engine: storepb.Engine_MYSQL,+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,+ plan: `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`,+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;",+ wantContent: `failed to get row count for "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;": table "t2" in the plan has no row estimate`,+ wantCode: code.Internal,+ fallback: true,+ },+ {+ name: "insert select reads the source plan",+ engine: storepb.Engine_MYSQL,+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,+ plan: testMySQLInsertSelectPlan,+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ wantContent: `"INSERT INTO t2 SELECT * FROM t WHERE d = 3;" inserts 100 rows. The count exceeds 5.`,+ wantCode: code.InsertTooManyRows,+ },+ {+ name: "insert table",+ engine: storepb.Engine_MYSQL,+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,+ plan: testMySQLInsertTablePlan,+ statement: "INSERT INTO t2 TABLE t;",+ wantContent: `"INSERT INTO t2 TABLE t;" inserts 1000 rows. The count exceeds 5.`,+ wantCode: code.InsertTooManyRows,+ },+ {+ name: "insert select capped by LIMIT",+ engine: storepb.Engine_MYSQL,ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,+ plan: testMySQLInsertSelectPlan,+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 3;",},} {t.Run(tc.name, func(t *testing.T) {- adviceList, err := advisor.SQLReviewCheck(context.Background(), sm, tc.statement, []*storepb.SQLReviewRule{- {- Type: tc.ruleType,- Level: storepb.SQLReviewRule_WARNING,- Engine: storepb.Engine_MYSQL,- Payload: &storepb.SQLReviewRule_NumberPayload{- NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{- Number: 5,- },- },- },- }, advisor.Context{- DBType: storepb.Engine_MYSQL,- Driver: db,- NoAppendBuiltin: true,- })- require.NoError(t, err)- require.Empty(t, adviceList)+ db := openTestMySQLAdvisorDB(t, tc.plan)+ testMySQLAdvisorSetErr = tc.setErr++ adviceList := checkTestRowLimitRule(t, db, tc.engine, tc.ruleType, storepb.SQLReviewRule_WARNING, tc.statement)+ if tc.wantContent == "" {+ require.Empty(t, adviceList)+ } else {+ require.Len(t, adviceList, 1)+ require.Equal(t, tc.wantCode.Int32(), adviceList[0].Code)+ require.Equal(t, tc.wantContent, adviceList[0].Content)+ }++ explained := tc.explained+ if explained == "" {+ explained = tc.statement+ }+ // The rule sets the JSON format version on the connection that runs the JSON EXPLAIN and+ // resets it where the server has the variable.+ wantQueries := []string{"SET SESSION explain_json_format_version = 1", "EXPLAIN FORMAT=JSON " + explained}+ if tc.setErr == nil {+ wantQueries = append(wantQueries, "SET SESSION explain_json_format_version = DEFAULT")+ }+ jsonQueries := len(wantQueries)+ if tc.fallback {+ wantQueries = append(wantQueries, "EXPLAIN FORMAT=TRADITIONAL "+explained)+ }+ require.Equal(t, wantQueries, testMySQLAdvisorQueries)+ for _, conn := range testMySQLAdvisorQueryConns[:jsonQueries] {+ require.Equal(t, testMySQLAdvisorQueryConns[0], conn)+ }})}}+func TestMySQLRowLimitAdvisorsWarnAboutStatementsBeyondExplainLimit(t *testing.T) {+ t.Run("affected row limit", func(t *testing.T) {+ db := openTestMySQLAdvisorDB(t, testMySQLSingleTableUpdatePlan)+ statement := strings.Repeat("UPDATE td SET c = 1;\n", common.MaximumLintExplainSize+2)++ adviceList := checkTestRowLimitRule(t, db, storepb.Engine_MYSQL, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, statement)+ require.Len(t, adviceList, common.MaximumLintExplainSize+1)+ for _, advice := range adviceList[:common.MaximumLintExplainSize] {+ require.Equal(t, storepb.Advice_ERROR, advice.Status)+ require.Contains(t, advice.Content, "affected 1000 rows")+ }+ warning := adviceList[common.MaximumLintExplainSize]+ require.Equal(t, storepb.Advice_WARNING, warning.Status)+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), warning.Code)+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT.String(), warning.Title)+ require.Equal(t, "Only the first 10 statements were estimated; 2 more were not checked against the row limit.", warning.Content)+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())+ require.Equal(t, common.MaximumLintExplainSize, countTestExplainQueries())+ })++ t.Run("insert row limit keeps counting VALUES rows", func(t *testing.T) {+ db := openTestMySQLAdvisorDB(t, testMySQLInsertSelectPlan)+ statement := strings.Repeat("INSERT INTO t2 SELECT * FROM t WHERE d = 3;\n", common.MaximumLintExplainSize+1) ++ "INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);"++ adviceList := checkTestRowLimitRule(t, db, storepb.Engine_MYSQL, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, statement)+ require.Len(t, adviceList, common.MaximumLintExplainSize+2)+ require.Equal(t, `"INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);" inserts 6 rows. The count exceeds 5.`, adviceList[common.MaximumLintExplainSize].Content)+ warning := adviceList[common.MaximumLintExplainSize+1]+ require.Equal(t, storepb.Advice_WARNING, warning.Status)+ require.Equal(t, code.InsertTooManyRows.Int32(), warning.Code)+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT.String(), warning.Title)+ require.Equal(t, "Only the first 10 statements were estimated; 1 more were not checked against the row limit.", warning.Content)+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())+ require.Equal(t, common.MaximumLintExplainSize, countTestExplainQueries())+ })+}++func countTestExplainQueries() int {+ count := 0+ for _, query := range testMySQLAdvisorQueries {+ if strings.HasPrefix(query, "EXPLAIN ") {+ count+++ }+ }+ return count+}+func TestMariaDBPriorBackupCheckAdvisor(t *testing.T) {sm := sheet.NewManager()diff --git a/backend/plugin/advisor/mysql/rule_insert_row_limit.go b/backend/plugin/advisor/mysql/rule_insert_row_limit.goindex 9ae5f1c020..8ad5558b0b 100644--- a/backend/plugin/advisor/mysql/rule_insert_row_limit.go+++ b/backend/plugin/advisor/mysql/rule_insert_row_limit.go@@ -3,7 +3,6 @@ package mysqlimport ("context""fmt"- "strconv""strings""github.com/bytebase/omni/mysql/ast"@@ -13,6 +12,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor""github.com/bytebase/bytebase/backend/plugin/advisor/code"+ mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")@@ -43,8 +43,9 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.ContexmaxRow := int(numberPayload.Number)driver := checkCtx.Driver+ title := checkCtx.Rule.Type.String()var advice []*storepb.Advice- explainCount := 0+ var explains advisor.ExplainBudgetfor _, stmt := range checkCtx.ParsedStatements {if stmt.AST == nil {@@ -61,45 +62,47 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.ContexbaseLine := stmt.BaseLine()text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"- line := baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line)+ position := common.ConvertANTLRLineToPosition(baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line))- // INSERT ... SELECT: use EXPLAIN to count rows.- if ins.Select != nil {+ // INSERT ... SELECT and INSERT ... TABLE: use EXPLAIN to count rows.+ if ins.Select != nil || ins.TableSource != nil {if driver == nil {continue}- explainCount++- res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN %s", text))+ if !explains.Spend(position) {+ continue+ }++ plan, err := mysqldriver.ExplainJSON(ctx, driver, text)if err != nil {advice = append(advice, &storepb.Advice{Status: level,Code: code.InsertTooManyRows.Int32(),- Title: checkCtx.Rule.Type.String(),+ Title: title,Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(line),+ StartPosition: position,+ })+ continue+ }+ rowCount, err := mysqldriver.EstimateAffectedRows(ctx, driver, ins, text, plan)+ if err != nil {+ advice = append(advice, &storepb.Advice{+ Status: level,+ Code: code.Internal.Int32(),+ Title: title,+ Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),+ StartPosition: position,})- } else {- rowCount, err := getInsertRows(res)- if err != nil {- advice = append(advice, &storepb.Advice{- Status: level,- Code: code.Internal.Int32(),- Title: checkCtx.Rule.Type.String(),- Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(line),- })- } else if rowCount = capRowsByLimit(rowCount, ins.Select.Limit); rowCount > int64(maxRow) {- advice = append(advice, &storepb.Advice{- Status: level,- Code: code.InsertTooManyRows.Int32(),- Title: checkCtx.Rule.Type.String(),- Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, rowCount, maxRow),- StartPosition: common.ConvertANTLRLineToPosition(line),- })- }+ continue}- if explainCount >= common.MaximumLintExplainSize {- break+ if rowCount > int64(maxRow) {+ advice = append(advice, &storepb.Advice{+ Status: level,+ Code: code.InsertTooManyRows.Int32(),+ Title: title,+ Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, rowCount, maxRow),+ StartPosition: position,+ })}continue}@@ -109,14 +112,14 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contexadvice = append(advice, &storepb.Advice{Status: level,Code: code.InsertTooManyRows.Int32(),- Title: checkCtx.Rule.Type.String(),+ Title: title,Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, len(ins.Values), maxRow),- StartPosition: common.ConvertANTLRLineToPosition(line),+ StartPosition: position,})}}- return advice, nil+ return explains.AppendSkippedAdvice(advice, title, code.InsertTooManyRows), nil}func contentStartIndex(text string) int {@@ -127,70 +130,3 @@ func contentStartIndex(text string) int {}return 0}--func capRowsByLimit(rowCount int64, limit *ast.Limit) int64 {- limitCount, ok := limitRowCount(limit)- if !ok || limitCount >= rowCount {- return rowCount- }- return limitCount-}--func limitRowCount(limit *ast.Limit) (int64, bool) {- if limit == nil || limit.Count == nil {- return 0, false- }- lit, ok := limit.Count.(*ast.IntLit)- if !ok || lit.Value < 0 {- return 0, false- }- return lit.Value, true-}--func getInsertRows(res []any) (int64, error) {- // the res struct is []any{columnName, columnTable, rowDataList}- if len(res) != 3 {- return 0, errors.Errorf("expected 3 but got %d", len(res))- }- columns, ok := res[0].([]string)- if !ok {- return 0, errors.Errorf("expected []string but got %t", res[0])- }- rowList, ok := res[2].([]any)- if !ok {- return 0, errors.Errorf("expected []any but got %t", res[2])- }- if len(rowList) < 1 {- return 0, errors.Errorf("not found any data")- }-- rowsIndex, err := getColumnIndex(columns, "rows")- if err != nil {- return 0, errors.Errorf("failed to find rows column")- }-- for _, rowAny := range rowList {- row, ok := rowAny.([]any)- if !ok {- return 0, errors.Errorf("expected []any but got %t", row)- }- switch col := row[rowsIndex].(type) {- case int:- return int64(col), nil- case int32:- return int64(col), nil- case int64:- return col, nil- case string:- v, err := strconv.ParseInt(col, 10, 64)- if err != nil {- return 0, errors.Errorf("expected int or int64 but got string(%s)", col)- }- return v, nil- default:- continue- }- }-- return 0, nil-}diff --git a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.goindex fa88b062aa..e90388d107 100644--- a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go+++ b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go@@ -3,7 +3,6 @@ package mysqlimport ("context""fmt"- "strconv""strings""github.com/bytebase/omni/mysql/ast"@@ -13,6 +12,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor""github.com/bytebase/bytebase/backend/plugin/advisor/code"+ mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")@@ -42,157 +42,68 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advmaxRow := int(numberPayload.Number)driver := checkCtx.Driver+ if driver == nil {+ return nil, nil+ }title := checkCtx.Rule.Type.String()var advice []*storepb.Advice- explainCount := 0-- if driver != nil {- for _, stmt := range checkCtx.ParsedStatements {- if stmt.AST == nil {- continue- }- node, ok := mysqlparser.GetOmniNode(stmt.AST)- if !ok {- continue- }-- var limit *ast.Limit- switch stmt := node.(type) {- case *ast.UpdateStmt:- limit = stmt.Limit- case *ast.DeleteStmt:- limit = stmt.Limit- default:- continue- }-- baseLine := stmt.BaseLine()- text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"- line := baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line)+ var explains advisor.ExplainBudget- explainCount++- // Prefer the JSON query plan: the tabular EXPLAIN's first "rows" value may- // be the scan estimate of a driving table unrelated to the DML target- // (BYT-9858). Fall back to the tabular heuristic when the JSON plan yields- // no flagged target node.- var rowCount int64- counted := false- if res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN FORMAT=JSON %s", text)); err == nil {- rowCount, counted = getRowsFromJSONPlan(res)- }- if !counted {- res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN %s", text))- if err != nil {- advice = append(advice, &storepb.Advice{- Status: level,- Code: code.StatementAffectedRowExceedsLimit.Int32(),- Title: title,- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(line),- })- } else if rowCount, err = getRows(res); err != nil {- advice = append(advice, &storepb.Advice{- Status: level,- Code: code.Internal.Int32(),- Title: title,- Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(line),- })- } else {- counted = true- }- }- if counted {- if rowCount = capRowsByLimit(rowCount, limit); rowCount > int64(maxRow) {- advice = append(advice, &storepb.Advice{- Status: level,- Code: code.StatementAffectedRowExceedsLimit.Int32(),- Title: title,- Content: fmt.Sprintf("\"%s\" affected %d rows (estimated). The count exceeds %d.", text, rowCount, maxRow),- StartPosition: common.ConvertANTLRLineToPosition(line),- })- }- }-- if explainCount >= common.MaximumLintExplainSize {- break- }+ for _, stmt := range checkCtx.ParsedStatements {+ if stmt.AST == nil {+ continue}- }-- return advice, nil-}--// getRowsFromJSONPlan extracts the affected-row estimate from an-// `EXPLAIN FORMAT=JSON` query result, whose single column holds the JSON plan.-func getRowsFromJSONPlan(res []any) (int64, bool) {- if len(res) != 3 {- return 0, false- }- rowList, ok := res[2].([]any)- if !ok {- return 0, false- }- var plan strings.Builder- for _, rowAny := range rowList {- row, ok := rowAny.([]any)+ node, ok := mysqlparser.GetOmniNode(stmt.AST)if !ok {- return 0, false+ continue}- for _, col := range row {- if s, ok := col.(string); ok {- plan.WriteString(s)- }+ switch node.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ default:+ continue}- }- return mysqlparser.GetEstimatedAffectedRowsFromExplainJSON(plan.String())-}-func getRows(res []any) (int64, error) {- // the res struct is []any{columnName, columnTable, rowDataList}- if len(res) != 3 {- return 0, errors.Errorf("expected 3 but got %d", len(res))- }- columns, ok := res[0].([]string)- if !ok {- return 0, errors.Errorf("expected []string but got %t", res[0])- }- rowList, ok := res[2].([]any)- if !ok {- return 0, errors.Errorf("expected []any but got %t", res[2])- }- if len(rowList) < 1 {- return 0, errors.Errorf("not found any data")- }+ baseLine := stmt.BaseLine()+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"+ position := common.ConvertANTLRLineToPosition(baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line))- rowsIndex, err := getColumnIndex(columns, "rows")- if err != nil {- return 0, errors.Errorf("failed to find rows column")- }-- for _, rowAny := range rowList {- row, ok := rowAny.([]any)- if !ok {- return 0, errors.Errorf("expected []any but got %t", row)+ if !explains.Spend(position) {+ continue}- switch col := row[rowsIndex].(type) {- case int:- return int64(col), nil- case int32:- return int64(col), nil- case int64:- return col, nil- case string:- v, err := strconv.ParseInt(col, 10, 64)- if err != nil {- return 0, errors.Errorf("expected int or int64 but got string(%s)", col)- }- return v, nil- default:+ query := mysqlparser.AffectedRowsQuery(node, stmt.Text)+ plan, err := mysqldriver.ExplainJSON(ctx, driver, query)+ if err != nil {+ advice = append(advice, &storepb.Advice{+ Status: level,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),+ StartPosition: position,+ })continue}+ rowCount, err := mysqldriver.EstimateAffectedRows(ctx, driver, node, query, plan)+ if err != nil {+ advice = append(advice, &storepb.Advice{+ Status: level,+ Code: code.Internal.Int32(),+ Title: title,+ Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),+ StartPosition: position,+ })+ continue+ }+ if rowCount > int64(maxRow) {+ advice = append(advice, &storepb.Advice{+ Status: level,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: fmt.Sprintf("\"%s\" affected %d rows (estimated). The count exceeds %d.", text, rowCount, maxRow),+ StartPosition: position,+ })+ }}- return 0, nil+ return explains.AppendSkippedAdvice(advice, title, code.StatementAffectedRowExceedsLimit), nil}diff --git a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.goindex eae8b96c4f..ba3d970d82 100644--- a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go+++ b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go@@ -61,25 +61,18 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contexcontinue}checker.checkInsert(stmt.Text, stmt.BaseLine(), insert)- if checker.explainCount >= common.MaximumLintExplainSize {- break- }}- return checker.generateAdvice()+ return checker.explains.AppendSkippedAdvice(checker.adviceList, checker.title, code.InsertTooManyRows), nil}type insertRowLimitChecker struct {- adviceList []*storepb.Advice- level storepb.Advice_Status- title string- maxRow int- driver *sql.DB- ctx context.Context- explainCount int-}--func (checker *insertRowLimitChecker) generateAdvice() ([]*storepb.Advice, error) {- return checker.adviceList, nil+ adviceList []*storepb.Advice+ level storepb.Advice_Status+ title string+ maxRow int+ driver *sql.DB+ ctx context.Context+ explains advisor.ExplainBudget}func (checker *insertRowLimitChecker) checkInsert(text string, baseLine int, stmt *ast.InsertStmt) {@@ -94,8 +87,10 @@ func (checker *insertRowLimitChecker) handleInsertQueryExpression(text string, lif checker.driver == nil {return}+ if !checker.explains.Spend(common.ConvertANTLRLineToPosition(line)) {+ return+ }- checker.explainCount++res, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_OCEANBASE, fmt.Sprintf("EXPLAIN format=json %s", text))if err != nil {checker.adviceList = append(checker.adviceList, &storepb.Advice{diff --git a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.go b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.gonew file mode 100644index 0000000000..bc7ce0b78d--- /dev/null+++ b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.go@@ -0,0 +1,32 @@+package oceanbase++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++func TestInsertRowLimitAdvisorKeepsCountingValuesBeyondExplainLimit(t *testing.T) {+ const insertPlan = `{"ID":0,"OPERATOR":"DISTRIBUTED INSERT","NAME":"","EST.ROWS":100,"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t","EST.ROWS":100}}`+ statement := strings.Repeat("INSERT INTO t2 SELECT * FROM t;\n", common.MaximumLintExplainSize+1) ++ "INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);"++ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, insertPlan, statement)+ require.Len(t, adviceList, common.MaximumLintExplainSize+2)+ for _, advice := range adviceList[:common.MaximumLintExplainSize] {+ require.Contains(t, advice.Content, "inserts 100 rows. The count exceeds 5.")+ }+ require.Contains(t, adviceList[common.MaximumLintExplainSize].Content, "inserts 6 rows. The count exceeds 5.")+ warning := adviceList[common.MaximumLintExplainSize+1]+ require.Equal(t, storepb.Advice_WARNING, warning.Status)+ require.Equal(t, code.InsertTooManyRows.Int32(), warning.Code)+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT.String(), warning.Title)+ require.Equal(t, "Only the first 10 statements were estimated; 1 more were not checked against the row limit.", warning.Content)+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())+ require.Equal(t, common.MaximumLintExplainSize, testOceanBaseExplainCount)+}diff --git a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.goindex ab15b10e74..40c836ecf5 100644--- a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go+++ b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go@@ -62,27 +62,26 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advchecker.handleStmt(stmt.Text, omniLine(stmt.BaseLine(), stmt.Text, n.Loc))default:}- if checker.explainCount >= common.MaximumLintExplainSize {- break- }}}- return checker.adviceList, nil+ return checker.explains.AppendSkippedAdvice(checker.adviceList, checker.title, code.StatementAffectedRowExceedsLimit), nil}type statementAffectedRowLimitChecker struct {- adviceList []*storepb.Advice- level storepb.Advice_Status- title string- maxRow int- driver *sql.DB- ctx context.Context- explainCount int+ adviceList []*storepb.Advice+ level storepb.Advice_Status+ title string+ maxRow int+ driver *sql.DB+ ctx context.Context+ explains advisor.ExplainBudget}func (checker *statementAffectedRowLimitChecker) handleStmt(text string, lineNumber int) {- checker.explainCount+++ if !checker.explains.Spend(common.ConvertANTLRLineToPosition(lineNumber)) {+ return+ }res, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_OCEANBASE, fmt.Sprintf("EXPLAIN format=json %s", text))if err != nil {checker.adviceList = append(checker.adviceList, &storepb.Advice{diff --git a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.go b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.gonew file mode 100644index 0000000000..032b7788cc--- /dev/null+++ b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.go@@ -0,0 +1,43 @@+package oceanbase++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++func TestStatementAffectedRowLimitAdvisor(t *testing.T) {+ const updatePlan = `{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":"","EST.ROWS":1000,"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t","EST.ROWS":1000}}`++ t.Run("estimate over the limit", func(t *testing.T) {+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, updatePlan, "UPDATE t SET c = 1;")+ require.Len(t, adviceList, 1)+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), adviceList[0].Code)+ require.Equal(t, `"UPDATE t SET c = 1;" affected 1000 rows (estimated). The count exceeds 5.`, adviceList[0].Content)+ })++ t.Run("missing estimate", func(t *testing.T) {+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, `{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":""}`, "UPDATE t SET c = 1;")+ require.Len(t, adviceList, 1)+ require.Equal(t, code.Internal.Int32(), adviceList[0].Code)+ require.Contains(t, adviceList[0].Content, `operator "DISTRIBUTED UPDATE" has no EST.ROWS`)+ })++ t.Run("statements beyond the EXPLAIN limit", func(t *testing.T) {+ statement := strings.Repeat("UPDATE t SET c = 1;\n", common.MaximumLintExplainSize+2)+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, updatePlan, statement)+ require.Len(t, adviceList, common.MaximumLintExplainSize+1)+ warning := adviceList[common.MaximumLintExplainSize]+ require.Equal(t, storepb.Advice_WARNING, warning.Status)+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), warning.Code)+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT.String(), warning.Title)+ require.Equal(t, "Only the first 10 statements were estimated; 2 more were not checked against the row limit.", warning.Content)+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())+ require.Equal(t, common.MaximumLintExplainSize, testOceanBaseExplainCount)+ })+}diff --git a/backend/plugin/advisor/oceanbase/common.go b/backend/plugin/advisor/oceanbase/common.goindex 858ab49da5..360018e48a 100644--- a/backend/plugin/advisor/oceanbase/common.go+++ b/backend/plugin/advisor/oceanbase/common.go@@ -1,36 +1,26 @@package oceanbaseimport (- "encoding/json"+ "strings""github.com/pkg/errors"-)-// ExplainJSON represents the JSON structure returned by EXPLAIN format=json-// We only care about the EST.ROWS field for row count estimation-type ExplainJSON struct {- EstRows int64 `json:"EST.ROWS"`-}+ mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"+)-// getEstimatedRowsFromJSON extracts the estimated row count from OceanBase EXPLAIN format=json result+// getEstimatedRowsFromJSON extracts the estimated row count from an OceanBase+// `EXPLAIN FORMAT=JSON` result shaped as []any{columnNames, columnTypes, rows}.func getEstimatedRowsFromJSON(res []any) (int64, error) {- // For EXPLAIN format=json, OceanBase returns JSON data- // The res struct is []any{columnName, columnTable, rowDataList}if len(res) < 3 {return 0, errors.Errorf("expected at least 3 elements but got %d", len(res))}-rowList, ok := res[2].([]any)if !ok {return 0, errors.Errorf("expected []any for row data but got %T", res[2])}- if len(rowList) == 0 {- return 0, errors.Errorf("no data returned from EXPLAIN")- }- // OceanBase might return JSON data split across multiple elements- // We need to concatenate them to form a valid JSON string- var jsonStr string+ // OceanBase splits the JSON plan across rows.+ var plan strings.Builderfor _, rowAny := range rowList {row, ok := rowAny.([]any)if !ok {@@ -38,20 +28,9 @@ func getEstimatedRowsFromJSON(res []any) (int64, error) {}for _, cellAny := range row {if cell, ok := cellAny.(string); ok {- jsonStr += cell+ plan.WriteString(cell)}}}-- if jsonStr == "" {- return 0, errors.Errorf("no JSON data found in EXPLAIN result")- }-- var explainData ExplainJSON- if err := json.Unmarshal([]byte(jsonStr), &explainData); err != nil {- return 0, errors.Errorf("failed to parse JSON: %v", err)- }-- // Return the estimated rows from the JSON response- return explainData.EstRows, nil+ return mysqlparser.EstimateAffectedRowsFromOceanBaseExplainJSON(plan.String())}diff --git a/backend/plugin/advisor/oceanbase/common_test.go b/backend/plugin/advisor/oceanbase/common_test.gonew file mode 100644index 0000000000..73d1ddb6fe--- /dev/null+++ b/backend/plugin/advisor/oceanbase/common_test.go@@ -0,0 +1,133 @@+package oceanbase++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "io"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/component/sheet"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor"+)++var (+ // testOceanBaseExplainPlan is the plan the fake OceanBase server returns for every EXPLAIN.+ testOceanBaseExplainPlan string+ // testOceanBaseExplainCount counts the EXPLAIN statements the fake server receives.+ testOceanBaseExplainCount int+)++func init() {+ sql.Register("test_oceanbase_advisor_explain", testOceanBaseExplainDriver{})+}++type testOceanBaseExplainDriver struct{}++func (testOceanBaseExplainDriver) Open(string) (driver.Conn, error) {+ return testOceanBaseExplainConn{}, nil+}++type testOceanBaseExplainConn struct{}++func (testOceanBaseExplainConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (testOceanBaseExplainConn) Close() error {+ return nil+}++func (testOceanBaseExplainConn) Begin() (driver.Tx, error) {+ return testOceanBaseExplainTx{}, nil+}++func (testOceanBaseExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+ if strings.HasPrefix(query, "EXPLAIN ") {+ testOceanBaseExplainCount+++ }+ return &testOceanBaseExplainRows{rows: [][]driver.Value{{testOceanBaseExplainPlan}}}, nil+}++type testOceanBaseExplainTx struct{}++func (testOceanBaseExplainTx) Commit() error {+ return nil+}++func (testOceanBaseExplainTx) Rollback() error {+ return nil+}++type testOceanBaseExplainRows struct {+ rows [][]driver.Value+ idx int+}++func (*testOceanBaseExplainRows) Columns() []string {+ return []string{"Query Plan"}+}++func (*testOceanBaseExplainRows) Close() error {+ return nil+}++func (r *testOceanBaseExplainRows) Next(dest []driver.Value) error {+ if r.idx >= len(r.rows) {+ return io.EOF+ }+ copy(dest, r.rows[r.idx])+ r.idx+++ return nil+}++// checkTestRowLimitRule runs one OceanBase row limit rule with a limit of 5 against the fake+// server, which answers every EXPLAIN with plan.+func checkTestRowLimitRule(t *testing.T, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, plan, statement string) []*storepb.Advice {+ t.Helper()+ testOceanBaseExplainPlan = plan+ testOceanBaseExplainCount = 0+ db, err := sql.Open("test_oceanbase_advisor_explain", "")+ require.NoError(t, err)+ t.Cleanup(func() { _ = db.Close() })++ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{+ {+ Type: ruleType,+ Level: level,+ Engine: storepb.Engine_OCEANBASE,+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: 5}},+ },+ }, advisor.Context{+ DBType: storepb.Engine_OCEANBASE,+ Driver: db,+ NoAppendBuiltin: true,+ })+ require.NoError(t, err)+ return adviceList+}++func TestGetEstimatedRowsFromJSON(t *testing.T) {+ // OceanBase splits the JSON plan across rows, and EST.ROWS may be fractional.+ rows, err := getEstimatedRowsFromJSON([]any{+ []string{"Query Plan"},+ []string{"VARCHAR"},+ []any{+ []any{`{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE",`},+ []any{`"NAME":"","EST.ROWS":41.6,"output":""}`},+ },+ })+ require.NoError(t, err)+ require.Equal(t, int64(42), rows)++ _, err = getEstimatedRowsFromJSON([]any{+ []string{"Query Plan"},+ []string{"VARCHAR"},+ []any{[]any{`{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":""}`}},+ })+ require.ErrorContains(t, err, `operator "DISTRIBUTED UPDATE" has no EST.ROWS`)+}diff --git a/backend/plugin/advisor/pg/advisor_insert_row_limit.go b/backend/plugin/advisor/pg/advisor_insert_row_limit.goindex fbc561527f..4bedb8fc9f 100644--- a/backend/plugin/advisor/pg/advisor_insert_row_limit.go+++ b/backend/plugin/advisor/pg/advisor_insert_row_limit.go@@ -4,15 +4,16 @@ import ("context""database/sql""fmt"+ "strings""github.com/pkg/errors""github.com/bytebase/omni/pg/ast"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor"advisorcode "github.com/bytebase/bytebase/backend/plugin/advisor/code"+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")var (@@ -53,18 +54,19 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.ContexTenantMode: checkCtx.TenantMode,}- return RunRules(checkCtx.ParsedStatements, []OmniRule{rule}), nil+ adviceList := RunRules(checkCtx.ParsedStatements, []OmniRule{rule})+ return rule.explains.AppendSkippedAdvice(adviceList, rule.Title, advisorcode.InsertTooManyRows), nil}type insertRowLimitRule struct {OmniBaseRule- maxRow int- driver *sql.DB- ctx context.Context- explainCount int- preExecutions []string- TenantMode bool+ maxRow int+ driver *sql.DB+ ctx context.Context+ explains advisor.ExplainBudget+ settings sessionSettings+ TenantMode bool}func (*insertRowLimitRule) Name() string {@@ -72,21 +74,19 @@ func (*insertRowLimitRule) Name() string {}func (r *insertRowLimitRule) OnStatement(node ast.Node) {+ r.settings.add(node, r.TrimmedStmtText())+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)switch n := node.(type) {- case *ast.VariableSetStmt:- if omniIsRoleOrSearchPathSet(n) {- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())- }case *ast.InsertStmt:- r.checkInsert(n)+ r.checkInsert(n, text)default:}}-func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {+func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt, text string) {code := advisorcode.Okrows := int64(0)- statementText := r.TrimmedStmtText()+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")// Count VALUES rows if this is INSERT ... VALUES.if sel, ok := ins.SelectStmt.(*ast.SelectStmt); ok && sel.ValuesLists != nil {@@ -97,15 +97,14 @@ func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {}} else if ins.SelectStmt != nil && r.driver != nil {// For INSERT ... SELECT, use EXPLAIN.- if r.explainCount >= common.MaximumLintExplainSize {+ if !r.explains.Spend(&storepb.Position{Line: r.ContentStartLine() + int32(r.BaseLine)}) {return}- r.explainCount++res, err := advisor.Query(r.ctx, advisor.QueryContext{TenantMode: r.TenantMode,- PreExecutions: r.preExecutions,- }, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))+ PreExecutions: r.settings.statements(),+ }, r.driver, storepb.Engine_POSTGRES, getExplainSQL(statementText))if err != nil {r.AddAdvice(&storepb.Advice{@@ -121,7 +120,7 @@ func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {return}- rowCount, err := getAffectedRows(res)+ rowCount, err := getInsertedRows(res)if err != nil {r.AddAdvice(&storepb.Advice{Status: r.Level,diff --git a/backend/plugin/advisor/pg/advisor_insert_row_limit_test.go b/backend/plugin/advisor/pg/advisor_insert_row_limit_test.gonew file mode 100644index 0000000000..e43a04a595--- /dev/null+++ b/backend/plugin/advisor/pg/advisor_insert_row_limit_test.go@@ -0,0 +1,126 @@+package pg++import (+ "fmt"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++func TestInsertRowLimit(t *testing.T) {+ const title = "STATEMENT_INSERT_ROW_LIMIT"+ for _, tc := range []struct {+ name string+ statement string+ plans map[string]string+ wantExplained []string+ want []*storepb.Advice+ }{+ {+ name: "insert select on conflict",+ statement: "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING;",+ plans: map[string]string{+ "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING": modifyTablePlan(1000),+ },+ wantExplained: []string{"INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.InsertTooManyRows.Int32(),+ Title: title,+ Content: `The statement "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING" inserts 1000 rows. The count exceeds 100.`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ // The CTE's 5000 deleted rows are not inserted rows.+ name: "insert with a data-modifying CTE counts only the inserted rows",+ statement: "WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1;",+ plans: map[string]string{+ "WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1": `[{"Plan": {"Node Type": "ModifyTable", "Operation": "Insert", "Plan Rows": 0, "Plans": [+ {"Node Type": "ModifyTable", "Operation": "Delete", "Parent Relationship": "InitPlan", "Plan Rows": 5000, "Plans": [+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}+ ]},+ {"Node Type": "Limit", "Parent Relationship": "Outer", "Plan Rows": 1}+ ]}}]`,+ },+ wantExplained: []string{"WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1"},+ },+ {+ name: "insert select within the limit",+ statement: "INSERT INTO t2 SELECT * FROM t LIMIT 30;",+ plans: map[string]string{"INSERT INTO t2 SELECT * FROM t LIMIT 30": modifyTablePlan(30)},+ wantExplained: []string{"INSERT INTO t2 SELECT * FROM t LIMIT 30"},+ },+ {+ name: "dry run failure",+ statement: "INSERT INTO missing SELECT * FROM t;",+ wantExplained: []string{"INSERT INTO missing SELECT * FROM t"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.InsertTooManyRows.Int32(),+ Title: title,+ Content: `"INSERT INTO missing SELECT * FROM t" dry runs failed: relation in "INSERT INTO missing SELECT * FROM t" does not exist`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "plan without a ModifyTable node",+ statement: "INSERT INTO t2 SELECT * FROM t;",+ plans: map[string]string{"INSERT INTO t2 SELECT * FROM t": "Insert on t2 (cost=0.00..17.00 rows=0 width=0)"},+ wantExplained: []string{"INSERT INTO t2 SELECT * FROM t"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.Internal.Int32(),+ Title: title,+ Content: `failed to get row count for "INSERT INTO t2 SELECT * FROM t": failed to parse the EXPLAIN (FORMAT JSON) output: invalid character 'I' looking for beginning of value`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "insert values is counted without explain",+ statement: "INSERT INTO t2 VALUES (1), (2), (3) ON CONFLICT DO NOTHING;",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ db, fake := newExplainFakeDB(t, tc.plans)+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, 100, tc.statement)+ require.Equal(t, tc.wantExplained, fake.explainedStatements())+ require.Equal(t, tc.want, got)+ })+ }+}++func TestInsertRowLimitReportsStatementsBeyondExplainLimit(t *testing.T) {+ plans := map[string]string{}+ var statements []string+ for i := range 11 {+ statement := fmt.Sprintf("INSERT INTO t2 SELECT * FROM t WHERE id = %d", i)+ plans[statement] = modifyTablePlan(1)+ statements = append(statements, statement+";")+ }+ statements = append(statements, "INSERT INTO t2 VALUES (1), (2), (3);", "INSERT INTO t2 SELECT * FROM t;")+ db, fake := newExplainFakeDB(t, plans)++ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_ERROR, 2, strings.Join(statements, "\n"))+ require.Len(t, fake.explainedStatements(), 10)+ require.Equal(t, []*storepb.Advice{+ {+ Status: storepb.Advice_ERROR,+ Code: code.InsertTooManyRows.Int32(),+ Title: "STATEMENT_INSERT_ROW_LIMIT",+ Content: `The statement "INSERT INTO t2 VALUES (1), (2), (3)" inserts 3 rows. The count exceeds 2.`,+ StartPosition: &storepb.Position{Line: 12},+ },+ {+ Status: storepb.Advice_WARNING,+ Code: code.InsertTooManyRows.Int32(),+ Title: "STATEMENT_INSERT_ROW_LIMIT",+ Content: "Only the first 10 statements were estimated; 2 more were not checked against the row limit.",+ StartPosition: &storepb.Position{Line: 11},+ },+ }, got)+}diff --git a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.goindex da898f917a..66fae4d0d5 100644--- a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go+++ b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go@@ -4,15 +4,16 @@ import ("context""database/sql""fmt"+ "strings""github.com/pkg/errors""github.com/bytebase/omni/pg/ast"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor""github.com/bytebase/bytebase/backend/plugin/advisor/code"+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")var (@@ -23,11 +24,11 @@ func init() {advisor.Register(storepb.Engine_POSTGRES, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, &StatementAffectedRowLimitAdvisor{})}-// StatementAffectedRowLimitAdvisor is the advisor checking for UPDATE/DELETE affected row limit.+// StatementAffectedRowLimitAdvisor is the advisor checking for UPDATE/DELETE/MERGE affected row limit.type StatementAffectedRowLimitAdvisor struct {}-// Check checks for UPDATE/DELETE affected row limit.+// Check checks for UPDATE/DELETE/MERGE affected row limit, including data-modifying CTEs.func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {level, err := advisor.NewStatusBySQLReviewRuleLevel(checkCtx.Rule.Level)if err != nil {@@ -54,17 +55,18 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advtenantMode: checkCtx.TenantMode,}- return RunRules(checkCtx.ParsedStatements, []OmniRule{rule}), nil+ adviceList := RunRules(checkCtx.ParsedStatements, []OmniRule{rule})+ return rule.explains.AppendSkippedAdvice(adviceList, rule.Title, code.StatementAffectedRowExceedsLimit), nil}type statementAffectedRowLimitRule struct {OmniBaseRule- maxRow int- driver *sql.DB- ctx context.Context- explainCount int- preExecutions []string- tenantMode bool+ maxRow int+ driver *sql.DB+ ctx context.Context+ explains advisor.ExplainBudget+ settings sessionSettings+ tenantMode bool}func (*statementAffectedRowLimitRule) Name() string {@@ -72,35 +74,43 @@ func (*statementAffectedRowLimitRule) Name() string {}func (r *statementAffectedRowLimitRule) OnStatement(node ast.Node) {+ r.settings.add(node, r.TrimmedStmtText())+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)switch n := node.(type) {- case *ast.VariableSetStmt:- if omniIsRoleOrSearchPathSet(n) {- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())+ case *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:+ r.checkAffectedRows(text)+ case *ast.SelectStmt:+ if hasDataModifyingCTE(n.WithClause) {+ r.checkAffectedRows(text)+ }+ case *ast.InsertStmt:+ if hasDataModifyingCTE(n.WithClause) {+ r.checkAffectedRows(text)+ }+ case *ast.CreateTableAsStmt:+ if query, ok := n.Query.(*ast.SelectStmt); ok && hasDataModifyingCTE(query.WithClause) {+ r.checkAffectedRows(text)}- case *ast.UpdateStmt, *ast.DeleteStmt:- r.checkAffectedRows()default:}}-func (r *statementAffectedRowLimitRule) checkAffectedRows() {- if r.explainCount >= common.MaximumLintExplainSize {+func (r *statementAffectedRowLimitRule) checkAffectedRows(text string) {+ if !r.explains.Spend(&storepb.Position{Line: r.ContentStartLine() + int32(r.BaseLine)}) {return}- r.explainCount++-- statementText := r.TrimmedStmtText()+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")res, err := advisor.Query(r.ctx, advisor.QueryContext{TenantMode: r.tenantMode,- PreExecutions: r.preExecutions,- }, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))+ PreExecutions: r.settings.statements(),+ }, r.driver, storepb.Engine_POSTGRES, getExplainSQL(statementText))if err != nil {r.AddAdvice(&storepb.Advice{Status: r.Level,- Code: code.InsertTooManyRows.Int32(),+ Code: code.StatementAffectedRowExceedsLimit.Int32(),Title: r.Title,Content: fmt.Sprintf("\"%s\" dry runs failed: %s", statementText, err.Error()),StartPosition: &storepb.Position{diff --git a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.go b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.gonew file mode 100644index 0000000000..4bd4c3caf8--- /dev/null+++ b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.go@@ -0,0 +1,174 @@+package pg++import (+ "fmt"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++func TestStatementAffectedRowLimit(t *testing.T) {+ const title = "STATEMENT_AFFECTED_ROW_LIMIT"+ for _, tc := range []struct {+ name string+ statement string+ plans map[string]string+ wantExplained []string+ want []*storepb.Advice+ }{+ {+ name: "estimate from the subplan feeding the update, not its init plan",+ statement: "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag);",+ plans: map[string]string{+ "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [+ {"Node Type": "Aggregate", "Parent Relationship": "InitPlan", "Subplan Name": "InitPlan 1", "Plan Rows": 1},+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 3333}+ ]}}]`,+ },+ wantExplained: []string{"UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `The statement "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)" affected 3333 rows (estimated). The count exceeds 1000.`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "merge",+ statement: "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1;",+ plans: map[string]string{+ "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1": modifyTablePlan(10000),+ },+ wantExplained: []string{"MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `The statement "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1" affected 10000 rows (estimated). The count exceeds 1000.`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "data-modifying CTEs",+ statement: "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d;\nWITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d;\nCREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d;",+ plans: map[string]string{+ "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d": `[{"Plan": {"Node Type": "Aggregate", "Plan Rows": 1, "Plans": [+ {"Node Type": "ModifyTable", "Parent Relationship": "InitPlan", "Subplan Name": "CTE d", "Plan Rows": 5000, "Plans": [+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}+ ]},+ {"Node Type": "CTE Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}+ ]}}]`,+ "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d": modifyTablePlan(20000),+ "CREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d": modifyTablePlan(100),+ },+ wantExplained: []string{+ "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d",+ "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d",+ "CREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d",+ },+ want: []*storepb.Advice{+ {+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `The statement "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d" affected 5000 rows (estimated). The count exceeds 1000.`,+ StartPosition: &storepb.Position{Line: 1},+ },+ {+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `The statement "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d" affected 20000 rows (estimated). The count exceeds 1000.`,+ StartPosition: &storepb.Position{Line: 2},+ },+ },+ },+ {+ name: "select, read-only CTE, and insert are not explained",+ statement: "SELECT * FROM big;\nWITH x AS (SELECT id FROM s) SELECT * FROM x;\nINSERT INTO big_archive SELECT * FROM big;",+ },+ {+ name: "explain analyze runs the statement it explains",+ statement: "EXPLAIN (ANALYZE, BUFFERS) DELETE FROM big WHERE s_id <= 50;",+ plans: map[string]string{+ "DELETE FROM big WHERE s_id <= 50": modifyTablePlan(5000),+ },+ wantExplained: []string{"DELETE FROM big WHERE s_id <= 50"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `The statement "DELETE FROM big WHERE s_id <= 50" affected 5000 rows (estimated). The count exceeds 1000.`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "explain without analyze is not explained",+ statement: "EXPLAIN (ANALYZE off) DELETE FROM big;",+ },+ {+ name: "estimate within the limit",+ statement: "DELETE FROM big WHERE id = 1;",+ plans: map[string]string{"DELETE FROM big WHERE id = 1": modifyTablePlan(1)},+ wantExplained: []string{"DELETE FROM big WHERE id = 1"},+ },+ {+ name: "dry run failure",+ statement: "DELETE FROM missing;",+ wantExplained: []string{"DELETE FROM missing"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: title,+ Content: `"DELETE FROM missing" dry runs failed: relation in "DELETE FROM missing" does not exist`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ {+ name: "plan without a ModifyTable node",+ statement: "DELETE FROM big;",+ plans: map[string]string{"DELETE FROM big": `[{"Plan": {"Node Type": "Seq Scan", "Plan Rows": 10000}}]`},+ wantExplained: []string{"DELETE FROM big"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.Internal.Int32(),+ Title: title,+ Content: `failed to get row count for "DELETE FROM big": the plan has no ModifyTable node`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ db, fake := newExplainFakeDB(t, tc.plans)+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, 1000, tc.statement)+ require.Equal(t, tc.wantExplained, fake.explainedStatements())+ require.Equal(t, tc.want, got)+ })+ }+}++func TestStatementAffectedRowLimitReportsStatementsBeyondExplainLimit(t *testing.T) {+ plans := map[string]string{}+ var statements []string+ for i := range 12 {+ statement := fmt.Sprintf("UPDATE t SET v = 'y' WHERE id = %d", i)+ plans[statement] = modifyTablePlan(1)+ statements = append(statements, statement+";")+ }+ db, fake := newExplainFakeDB(t, plans)++ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, 1000, strings.Join(statements, "\n"))+ require.Len(t, fake.explainedStatements(), 10)+ require.Equal(t, []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementAffectedRowExceedsLimit.Int32(),+ Title: "STATEMENT_AFFECTED_ROW_LIMIT",+ Content: "Only the first 10 statements were estimated; 2 more were not checked against the row limit.",+ StartPosition: &storepb.Position{Line: 11},+ }}, got)+}diff --git a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.goindex 3844ee9347..5629aa3402 100644--- a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go+++ b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go@@ -4,6 +4,7 @@ import ("context""database/sql""fmt"+ "strings""github.com/bytebase/omni/pg/ast"@@ -11,6 +12,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor""github.com/bytebase/bytebase/backend/plugin/advisor/code"+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")var (@@ -58,11 +60,11 @@ func (*StatementDMLDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Cotype statementDMLDryRunRule struct {OmniBaseRule- driver *sql.DB- ctx context.Context- explainCount int- preExecutions []string- tenantMode bool+ driver *sql.DB+ ctx context.Context+ explainCount int+ settings sessionSettings+ tenantMode bool}// Name returns the rule name.@@ -71,18 +73,20 @@ func (*statementDMLDryRunRule) Name() string {}func (r *statementDMLDryRunRule) OnStatement(node ast.Node) {+ r.settings.add(node, r.TrimmedStmtText())+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)switch n := node.(type) {- case *ast.VariableSetStmt:- if omniIsRoleOrSearchPathSet(n) {- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:+ r.checkDMLDryRun(text)+ case *ast.SelectStmt:+ if hasDataModifyingCTE(n.WithClause) {+ r.checkDMLDryRun(text)}- case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:- r.checkDMLDryRun()default:}}-func (r *statementDMLDryRunRule) checkDMLDryRun() {+func (r *statementDMLDryRunRule) checkDMLDryRun(text string) {// Check if we've hit the maximum number of EXPLAIN queriesif r.explainCount >= common.MaximumLintExplainSize {return@@ -90,12 +94,12 @@ func (r *statementDMLDryRunRule) checkDMLDryRun() {r.explainCount++- statementText := r.TrimmedStmtText()+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")// Run EXPLAIN to perform dry run_, err := advisor.Query(r.ctx, advisor.QueryContext{TenantMode: r.tenantMode,- PreExecutions: r.preExecutions,+ PreExecutions: r.settings.statements(),}, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))if err != nil {diff --git a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.go b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.gonew file mode 100644index 0000000000..99b6d0216f--- /dev/null+++ b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.go@@ -0,0 +1,84 @@+package pg++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/component/sheet"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++func TestStatementDMLDryRun(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ plans map[string]string+ wantExplained []string+ want []*storepb.Advice+ }{+ {+ name: "insert, update, delete, and merge",+ statement: "INSERT INTO t VALUES (1);\nUPDATE t SET v = 1;\nDELETE FROM t;\nMERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE;",+ plans: map[string]string{+ "INSERT INTO t VALUES (1)": modifyTablePlan(1),+ "UPDATE t SET v = 1": modifyTablePlan(1),+ "DELETE FROM t": modifyTablePlan(1),+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE": modifyTablePlan(1),+ },+ wantExplained: []string{+ "INSERT INTO t VALUES (1)",+ "UPDATE t SET v = 1",+ "DELETE FROM t",+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE",+ },+ },+ {+ name: "data-modifying CTE",+ statement: "WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d;",+ plans: map[string]string{"WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d": modifyTablePlan(1)},+ wantExplained: []string{"WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d"},+ },+ {+ name: "explain analyze runs the statement it explains",+ statement: "EXPLAIN (ANALYZE, BUFFERS) INSERT INTO t VALUES (1);",+ plans: map[string]string{"INSERT INTO t VALUES (1)": modifyTablePlan(1)},+ wantExplained: []string{"INSERT INTO t VALUES (1)"},+ },+ {+ name: "select and explain without analyze are not dry run",+ statement: "SELECT * FROM t;\nEXPLAIN DELETE FROM t;",+ },+ {+ name: "dry run failure",+ statement: "MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE;",+ wantExplained: []string{"MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE"},+ want: []*storepb.Advice{{+ Status: storepb.Advice_WARNING,+ Code: code.StatementDMLDryRunFailed.Int32(),+ Title: "STATEMENT_DML_DRY_RUN",+ Content: `"MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE" dry runs failed: relation in "MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE" does not exist`,+ StartPosition: &storepb.Position{Line: 1},+ }},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ db, fake := newExplainFakeDB(t, tc.plans)+ got, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), tc.statement, []*storepb.SQLReviewRule{{+ Type: storepb.SQLReviewRule_STATEMENT_DML_DRY_RUN,+ Level: storepb.SQLReviewRule_WARNING,+ Engine: storepb.Engine_POSTGRES,+ }}, advisor.Context{+ DBType: storepb.Engine_POSTGRES,+ Driver: db,+ NoAppendBuiltin: true,+ })+ require.NoError(t, err)+ require.Equal(t, tc.wantExplained, fake.explainedStatements())+ require.Equal(t, tc.want, got)+ })+ }+}diff --git a/backend/plugin/advisor/pg/testutil_test.go b/backend/plugin/advisor/pg/testutil_test.gonew file mode 100644index 0000000000..b9d17299d8--- /dev/null+++ b/backend/plugin/advisor/pg/testutil_test.go@@ -0,0 +1,132 @@+package pg++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "fmt"+ "io"+ "strings"+ "sync"+ "testing"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/component/sheet"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor"+)++// explainFake stands in for PostgreSQL behind advisor.Query. It answers `EXPLAIN (FORMAT JSON)` or+// `EXPLAIN` with the plan registered for the statement and fails to explain any other statement.+type explainFake struct {+ plans map[string]string++ mu sync.Mutex+ explained []string+}++func newExplainFakeDB(t *testing.T, plans map[string]string) (*sql.DB, *explainFake) {+ t.Helper()+ fake := &explainFake{plans: plans}+ db := sql.OpenDB(fake)+ t.Cleanup(func() { require.NoError(t, db.Close()) })+ return db, fake+}++func (f *explainFake) Connect(context.Context) (driver.Conn, error) {+ return explainFakeConn{fake: f}, nil+}++func (f *explainFake) Driver() driver.Driver {+ return explainFakeDriver{fake: f}+}++func (f *explainFake) explainedStatements() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return f.explained+}++type explainFakeDriver struct{ fake *explainFake }++func (d explainFakeDriver) Open(string) (driver.Conn, error) {+ return explainFakeConn(d), nil+}++type explainFakeConn struct{ fake *explainFake }++func (explainFakeConn) Prepare(query string) (driver.Stmt, error) {+ return nil, errors.Errorf("unexpected prepare of %q", query)+}++func (explainFakeConn) Close() error { return nil }++func (explainFakeConn) Begin() (driver.Tx, error) { return explainFakeTx{}, nil }++func (explainFakeConn) ExecContext(context.Context, string, []driver.NamedValue) (driver.Result, error) {+ return driver.ResultNoRows, nil+}++func (c explainFakeConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+ statement, ok := strings.CutPrefix(query, "EXPLAIN (FORMAT JSON) ")+ if !ok {+ statement, ok = strings.CutPrefix(query, "EXPLAIN ")+ }+ if !ok {+ return nil, errors.Errorf("unexpected query %q", query)+ }+ c.fake.mu.Lock()+ c.fake.explained = append(c.fake.explained, statement)+ c.fake.mu.Unlock()+ plan, ok := c.fake.plans[statement]+ if !ok {+ return nil, errors.Errorf("relation in %q does not exist", statement)+ }+ return &explainFakeRows{plan: plan}, nil+}++type explainFakeTx struct{}++func (explainFakeTx) Commit() error { return nil }+func (explainFakeTx) Rollback() error { return nil }++type explainFakeRows struct {+ plan string+ done bool+}++func (*explainFakeRows) Columns() []string { return []string{"QUERY PLAN"} }++func (*explainFakeRows) Close() error { return nil }++func (r *explainFakeRows) Next(dest []driver.Value) error {+ if r.done {+ return io.EOF+ }+ dest[0] = r.plan+ r.done = true+ return nil+}++// modifyTablePlan returns a minimal `EXPLAIN (FORMAT JSON)` plan whose ModifyTable node is fed rows.+func modifyTablePlan(rows int) string {+ return fmt.Sprintf(`[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [{"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": %d}]}}]`, rows)+}++func runRowLimitRule(t *testing.T, db *sql.DB, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, limit int32, statement string) []*storepb.Advice {+ t.Helper()+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{{+ Type: ruleType,+ Level: level,+ Engine: storepb.Engine_POSTGRES,+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: limit}},+ }}, advisor.Context{+ DBType: storepb.Engine_POSTGRES,+ Driver: db,+ NoAppendBuiltin: true,+ })+ require.NoError(t, err)+ return adviceList+}diff --git a/backend/plugin/advisor/pg/utils.go b/backend/plugin/advisor/pg/utils.goindex 76d0a0c1fd..b7b5decb36 100644--- a/backend/plugin/advisor/pg/utils.go+++ b/backend/plugin/advisor/pg/utils.go@@ -3,11 +3,13 @@ package pgimport ("fmt""regexp"- "strconv"+ "slices""strings""github.com/bytebase/omni/pg/ast""github.com/pkg/errors"++ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")func getTemplateRegexp(template string, templateList []string, tokens map[string]string) (*regexp.Regexp, error) {@@ -28,45 +30,73 @@ func normalizeSchemaName(schemaName string) string {return schemaName}-// getAffectedRows extracts the estimated row count from a PostgreSQL EXPLAIN result.-func getAffectedRows(res []any) (int64, error) {+// getExplainSQL returns the query whose result getExplainPlan reads.+func getExplainSQL(statement string) string {+ return fmt.Sprintf("EXPLAIN (FORMAT JSON) %s", statement)+}++// getExplainPlan returns the JSON plan from the advisor.Query result of a getExplainSQL query.+func getExplainPlan(res []any) (string, error) {// the res struct is []any{columnName, columnTable, rowDataList}if len(res) != 3 {- return 0, errors.Errorf("expected 3 but got %d", len(res))+ return "", errors.Errorf("expected 3 but got %d", len(res))}rowList, ok := res[2].([]any)if !ok {- return 0, errors.Errorf("expected []any but got %t", res[2])- }- // EXPLAIN output has at least 2 rows- if len(rowList) < 2 {- return 0, errors.Errorf("not found any data")+ return "", errors.Errorf("expected []any but got %T", res[2])}- // We need row 2- rowTwo, ok := rowList[1].([]any)- if !ok {- return 0, errors.Errorf("expected []any but got %t", rowList[0])+ if len(rowList) != 1 {+ return "", errors.Errorf("expected one plan row but got %d", len(rowList))}- // PostgreSQL EXPLAIN result has one column- if len(rowTwo) != 1 {- return 0, errors.Errorf("expected one but got %d", len(rowTwo))+ row, ok := rowList[0].([]any)+ if !ok || len(row) != 1 {+ return "", errors.Errorf("expected one plan column but got %v", rowList[0])}- // Get the string value- text, ok := rowTwo[0].(string)+ plan, ok := row[0].(string)if !ok {- return 0, errors.Errorf("expected string but got %t", rowTwo[0])+ return "", errors.Errorf("expected string but got %T", row[0])}+ return plan, nil+}- rowsRegexp := regexp.MustCompile("rows=([0-9]+)")- matches := rowsRegexp.FindStringSubmatch(text)- if len(matches) != 2 {- return 0, errors.Errorf("failed to find rows in %q", text)+// getAffectedRows returns the estimated rows a statement and its data-modifying CTEs modify, from+// the advisor.Query result of its getExplainSQL query.+func getAffectedRows(res []any) (int64, error) {+ plan, err := getExplainPlan(res)+ if err != nil {+ return 0, err}- value, err := strconv.ParseInt(matches[1], 10, 64)+ return pgparser.GetEstimatedAffectedRowsFromExplainJSON(plan)+}++// getInsertedRows returns the estimated rows an INSERT adds, without the rows its data-modifying+// CTEs change, from the advisor.Query result of its getExplainSQL query.+func getInsertedRows(res []any) (int64, error) {+ plan, err := getExplainPlan(res)if err != nil {- return 0, errors.Errorf("failed to get integer from %q", matches[1])+ return 0, err+ }+ return pgparser.GetEstimatedInsertedRowsFromExplainJSON(plan)+}++// hasDataModifyingCTE reports whether a WITH clause contains INSERT, UPDATE, DELETE, or MERGE.+// PostgreSQL only allows data-modifying statements in a top-level WITH.+func hasDataModifyingCTE(with *ast.WithClause) bool {+ if with == nil || with.Ctes == nil {+ return false+ }+ for _, item := range with.Ctes.Items {+ cte, ok := item.(*ast.CommonTableExpr)+ if !ok {+ continue+ }+ switch cte.Ctequery.(type) {+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:+ return true+ default:+ }}- return value, nil+ return false}// nolint:unused@@ -153,14 +183,78 @@ func omniAlterTableCmds(alter *ast.AlterTableStmt) []*ast.AlterTableCmd {return cmds}-// nolint:unused-// omniIsRoleOrSearchPathSet checks if a VariableSetStmt is SET ROLE or SET search_path.+// sessionSettings holds, in order, the statements that set the role or search path, which an EXPLAIN+// replays. A SET LOCAL setting lasts until its transaction ends, ROLLBACK also drops the settings of+// its transaction, and DISCARD ALL drops every setting.+type sessionSettings struct {+ settings []sessionSetting+ // transactionStart is the number of settings when the current transaction began.+ transactionStart int+ inTransaction bool+}++type sessionSetting struct {+ name string+ text string+ local bool+}++// add records node, whose text is text, when it changes the settings.+func (s *sessionSettings) add(node ast.Node, text string) {+ switch n := node.(type) {+ case *ast.VariableSetStmt:+ if omniIsRoleOrSearchPathSet(n) {+ if n.Kind == ast.VAR_SET_CURRENT && !n.IsLocal {+ // SET ... FROM CURRENT keeps the value a SET LOCAL gave the setting after the transaction ends.+ for i := range s.settings {+ if strings.EqualFold(s.settings[i].name, n.Name) {+ s.settings[i].local = false+ }+ }+ }+ s.settings = append(s.settings, sessionSetting{name: n.Name, text: text, local: n.IsLocal})+ }+ case *ast.DiscardStmt:+ if n.Target == ast.DISCARD_ALL {+ s.settings, s.transactionStart = nil, 0+ }+ case *ast.TransactionStmt:+ switch n.Kind {+ case ast.TRANS_STMT_BEGIN, ast.TRANS_STMT_START:+ // A BEGIN inside a transaction only warns.+ if !s.inTransaction {+ s.transactionStart, s.inTransaction = len(s.settings), true+ }+ case ast.TRANS_STMT_COMMIT, ast.TRANS_STMT_ROLLBACK:+ // A ROLLBACK outside a transaction only warns.+ if n.Kind == ast.TRANS_STMT_ROLLBACK && s.inTransaction {+ s.settings = s.settings[:s.transactionStart]+ }+ s.settings = slices.DeleteFunc(s.settings, func(setting sessionSetting) bool { return setting.local })+ // COMMIT AND CHAIN and ROLLBACK AND CHAIN start the next transaction from here.+ s.transactionStart, s.inTransaction = len(s.settings), n.Chain+ default:+ }+ default:+ }+}++func (s *sessionSettings) statements() []string {+ var statements []string+ for _, setting := range s.settings {+ statements = append(statements, setting.text)+ }+ return statements+}++// omniIsRoleOrSearchPathSet checks if a VariableSetStmt sets the role or search path, including a+// RESET ALL.func omniIsRoleOrSearchPathSet(stmt *ast.VariableSetStmt) bool {if stmt == nil {return false}name := strings.ToLower(stmt.Name)- return name == "role" || name == "search_path"+ return name == "role" || name == "search_path" || stmt.Kind == ast.VAR_RESET_ALL}// nolint:unuseddiff --git a/backend/plugin/advisor/pg/utils_test.go b/backend/plugin/advisor/pg/utils_test.goindex 9cb7a54b02..651f91453f 100644--- a/backend/plugin/advisor/pg/utils_test.go+++ b/backend/plugin/advisor/pg/utils_test.go@@ -33,6 +33,11 @@ func TestOmniIsRoleOrSearchPathSet(t *testing.T) {sql: "SET search_path = myschema, public; UPDATE t SET a = 1;",want: []string{"SET search_path = myschema, public"},},+ {+ name: "reset all",+ sql: "SET search_path = myschema; RESET ALL; UPDATE t SET a = 1;",+ want: []string{"SET search_path = myschema", "RESET ALL"},+ },{name: "ignore unrelated set variable",sql: "SET statement_timeout = '1s'; DELETE FROM t;",@@ -65,3 +70,59 @@ func TestOmniIsRoleOrSearchPathSet(t *testing.T) {})}}++func TestSessionSettings(t *testing.T) {+ var settings sessionSettings+ add := func(statement string) {+ statements, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)+ require.NoError(t, err)+ for _, stmt := range statements {+ node, ok := pgparser.GetOmniNode(stmt.AST)+ require.True(t, ok)+ settings.add(node, strings.TrimRight(strings.TrimSpace(stmt.Text), ";"))+ }+ }++ add(`SET search_path = z;+DISCARD ALL;+SET search_path = a;+BEGIN;+SET LOCAL search_path = b;+SET ROLE r;+COMMIT;+BEGIN;+SET search_path = c;+ROLLBACK;+BEGIN;+SET search_path = d;+COMMIT AND CHAIN;+ROLLBACK;+SET LOCAL search_path = e;+SET statement_timeout = '1s';+RESET search_path;`)+ // The settings replay in statement order, so RESET follows the SET LOCAL it overrides.+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "SET LOCAL search_path = e", "RESET search_path"}, settings.statements())++ // A nested BEGIN only warns, so the ROLLBACK returns to the settings before the first BEGIN, and+ // the transaction's end also ends the SET LOCAL.+ add(`BEGIN;+SET search_path = f;+BEGIN;+SET search_path = g;+ROLLBACK;`)+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path"}, settings.statements())++ // A ROLLBACK outside a transaction only warns, so it keeps the session settings.+ add(`SET search_path = h;+SET LOCAL search_path = i;+ROLLBACK;`)+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path", "SET search_path = h"}, settings.statements())++ // SET ... FROM CURRENT keeps the SET LOCAL value of its own setting past COMMIT.+ add(`BEGIN;+SET LOCAL ROLE s;+SET LOCAL search_path = j;+SET search_path FROM CURRENT;+COMMIT;`)+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path", "SET search_path = h", "SET LOCAL search_path = j", "SET search_path FROM CURRENT"}, settings.statements())+}diff --git a/backend/plugin/advisor/utils.go b/backend/plugin/advisor/utils.goindex e8fa271329..d8fce879c2 100644--- a/backend/plugin/advisor/utils.go+++ b/backend/plugin/advisor/utils.go@@ -9,11 +9,50 @@ import ("github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code""github.com/bytebase/bytebase/backend/plugin/parser/base")+// ExplainBudget caps the statements a row limit rule EXPLAINs at common.MaximumLintExplainSize and+// counts the statements it leaves unchecked.+type ExplainBudget struct {+ used int+ skipped int+ firstSkipped *storepb.Position+}++// Spend reports whether the statement at position may be explained, recording it as skipped+// otherwise.+func (b *ExplainBudget) Spend(position *storepb.Position) bool {+ if b.used < common.MaximumLintExplainSize {+ b.used+++ return true+ }+ if b.skipped == 0 {+ b.firstSkipped = position+ }+ b.skipped+++ return false+}++// AppendSkippedAdvice appends one warning covering the skipped statements, if any. It is a warning+// whatever the rule's level.+func (b *ExplainBudget) AppendSkippedAdvice(adviceList []*storepb.Advice, title string, adviceCode code.Code) []*storepb.Advice {+ if b.skipped == 0 {+ return adviceList+ }+ return append(adviceList, &storepb.Advice{+ Status: storepb.Advice_WARNING,+ Code: adviceCode.Int32(),+ Title: title,+ Content: fmt.Sprintf("Only the first %d statements were estimated; %d more were not checked against the row limit.", common.MaximumLintExplainSize, b.skipped),+ StartPosition: b.firstSkipped,+ })+}+type QueryContext struct {TenantMode boolPreExecutions []string@@ -152,7 +191,8 @@ func ContainsDDL(engine storepb.Engine, parsedStatements []base.ParsedStatement)case storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED,storepb.StatementType_INSERT,storepb.StatementType_UPDATE,- storepb.StatementType_DELETE:+ storepb.StatementType_DELETE,+ storepb.StatementType_MERGE:default:return true}diff --git a/backend/plugin/db/cockroachdb/stats.go b/backend/plugin/db/cockroachdb/stats.goindex b4e15c0402..93766fdc42 100644--- a/backend/plugin/db/cockroachdb/stats.go+++ b/backend/plugin/db/cockroachdb/stats.go@@ -4,62 +4,223 @@ import ("context""database/sql""fmt"+ "math""regexp"+ "slices""strconv"+ "strings"+ "unicode/utf8""github.com/cockroachdb/cockroach-go/v2/crdb""github.com/pkg/errors"-)-var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")+ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)+// CountAffectedRows returns the optimizer's estimate of the rows the statement modifies. A search+// path that base.WithSearchPath put before the statement is set for the EXPLAIN.func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)- var rowCount int64- if err := crdb.Execute(func() error {- rows, err := d.db.QueryContext(ctx, explainSQL)+ setup, statement := base.SplitSearchPath(statement)+ var settings []string+ if setup != "" {+ settings = append(settings, setup)+ }+ plan, err := d.explain(ctx, statement, settings...)+ if err != nil {+ return 0, err+ }+ rows, err := getAffectedRowsFromPlan(plan)+ if err == nil || !hasDeleteRangeNode(plan) {+ return rows, err+ }+ // A delete range node has no row estimate, while the plan of the same DELETE without the fast+ // path has. Versions without the setting keep the original error.+ fallbackPlan, fallbackErr := d.explain(ctx, statement, append(settings, "SET LOCAL optimizer_use_delete_range_fast_path = off")...)+ if fallbackErr != nil {+ return 0, err+ }+ return getAffectedRowsFromPlan(fallbackPlan)+}++// explain returns the EXPLAIN output of the statement after the SET LOCAL settings, which run in a+// transaction that is rolled back so the pooled connection keeps its session.+func (d *Driver) explain(ctx context.Context, statement string, settings ...string) ([]string, error) {+ var plan []string+ err := crdb.Execute(func() error {+ if len(settings) == 0 {+ var err error+ plan, err = queryPlan(ctx, d.db.QueryContext, statement)+ return err+ }+ tx, err := d.db.BeginTx(ctx, nil)if err != nil {return err}- defer rows.Close()-- // test-bb=# EXPLAIN INSERT INTO t SELECT * FROM t;- // QUERY PLAN- // -------------------------------------------------------------- // Insert on t (cost=0.00..1.03 rows=0 width=0)- // -> Seq Scan on t t_1 (cost=0.00..1.03 rows=3 width=520)- // (2 rows)- //- // d1=# explain select * from h1;- // QUERY PLAN- // ------------------------------------------------------- // Seq Scan on h1 (cost=0.00..35.50 rows=2550 width=4)- // (1 row)- for rows.Next() {- var planRow sql.NullString- if err := rows.Scan(&planRow); err != nil {+ defer tx.Rollback()+ for _, setting := range settings {+ if _, err := tx.ExecContext(ctx, setting); err != nil {return err}+ }+ plan, err = queryPlan(ctx, tx.QueryContext, statement)+ return err+ })+ return plan, err+}- if !planRow.Valid {- continue- }+func queryPlan(ctx context.Context, query func(context.Context, string, ...any) (*sql.Rows, error), statement string) ([]string, error) {+ rows, err := query(ctx, fmt.Sprintf("EXPLAIN %s", statement))+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ var plan []string+ for rows.Next() {+ var line sql.NullString+ if err := rows.Scan(&line); err != nil {+ return nil, err+ }+ plan = append(plan, line.String)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return plan, nil+}- matches := rowsRegexp.FindStringSubmatch(planRow.String)- if len(matches) != 2 {- continue- }- v, err := strconv.ParseInt(matches[1], 10, 64)- if err != nil {- return errors.Errorf("failed to get integer from %q", matches[1])+type planNode struct {+ name string+ // column is the rune offset of the node's bullet, which grows with the node's depth.+ column int+ attributes []string+}++// parsePlanNodes returns the nodes of EXPLAIN text output in pre-order. A node line has a bullet+// after the tree drawing, and the lines before the next node line are the node's attributes.+func parsePlanNodes(plan []string) []planNode {+ var nodes []planNode+ for _, line := range plan {+ line = strings.TrimRight(line, " \t\r")+ text := strings.TrimLeft(line, " │├└─")+ if name, ok := strings.CutPrefix(text, "• "); ok {+ nodes = append(nodes, planNode{+ name: name,+ column: utf8.RuneCountInString(line[:len(line)-len(text)]),+ })+ continue+ }+ if len(nodes) > 0 && text != "" {+ nodes[len(nodes)-1].attributes = append(nodes[len(nodes)-1].attributes, text)+ }+ }+ return nodes+}++func hasDeleteRangeNode(plan []string) bool {+ return slices.ContainsFunc(parsePlanNodes(plan), func(node planNode) bool {+ return node.name == "delete range"+ })+}++var mutationNodeNames = map[string]bool{+ "insert": true,+ "insert fast path": true,+ "upsert": true,+ "update": true,+ "delete": true,+ "delete range": true,+}++// getAffectedRowsFromPlan sums the rows written by the mutation nodes in EXPLAIN text output,+// including the mutations of CTEs and statement sources. Foreign key cascades are left out, as+// CockroachDB leaves them out of the rows a statement affects. It fails when the plan has no+// mutation node or a mutation node has no row count.+func getAffectedRowsFromPlan(plan []string) (int64, error) {+ nodes := parsePlanNodes(plan)+ var total float64+ mutationCount := 0+ cascadeColumn := -1+ for i, node := range nodes {+ if cascadeColumn >= 0 && node.column > cascadeColumn {+ continue+ }+ cascadeColumn = -1+ if node.name == "fk-cascade" {+ cascadeColumn = node.column+ continue+ }+ if !mutationNodeNames[node.name] {+ continue+ }+ rows, ok := getMutationRows(nodes, i)+ if !ok {+ return 0, errors.Errorf("the %s node in the plan has no row count estimate", node.name)+ }+ total += rows+ mutationCount+++ }+ if mutationCount == 0 {+ return 0, errors.New("the plan has no mutation node")+ }+ return common.RoundRows(total), nil+}++// getMutationRows returns the rows written by the mutation node nodes[i]. A mutation that returns+// rows has its own estimate. Otherwise the first row count in its subtree, in pre-order, estimates+// its input, capped by the counts of the limit nodes above it, which have no estimate of their own.+func getMutationRows(nodes []planNode, i int) (float64, bool) {+ limit := math.Inf(1)+ for j := i; j < len(nodes); j++ {+ node := nodes[j]+ if j > i && node.column <= nodes[i].column {+ break+ }+ for _, attribute := range node.attributes {+ key, value, _ := strings.Cut(attribute, ": ")+ switch {+ case key == "estimated row count":+ rows, ok := parseEstimatedRowCount(value)+ return math.Min(rows, limit), ok+ case key == "size" && (node.name == "values" || node.name == "insert fast path"):+ rows, ok := parseValuesSize(value)+ return math.Min(rows, limit), ok+ case key == "count" && node.name == "limit":+ if count, ok := parseCount(value); ok {+ limit = math.Min(limit, count)+ }+ default:}- rowCount = v}- err = rows.Err()- return err- }); err != nil {- return 0, err}+ return 0, false+}- return rowCount, nil+// parseEstimatedRowCount parses "1,000 (10% of the table; stats collected 1 minute ago)". A node+// under a soft limit shows a "min - max" range, whose maximum leaves the limit to the nodes above.+func parseEstimatedRowCount(value string) (float64, bool) {+ value, _, _ = strings.Cut(value, " (")+ if _, maximum, ok := strings.Cut(value, " - "); ok {+ value = maximum+ }+ return parseCount(value)+}++var valuesSizeRegexp = regexp.MustCompile(`^[\d,]+ columns?, ([\d,]+) rows?$`)++// parseValuesSize parses the rows of "5 columns, 2 rows".+func parseValuesSize(value string) (float64, bool) {+ match := valuesSizeRegexp.FindStringSubmatch(value)+ if match == nil {+ return 0, false+ }+ return parseCount(match[1])+}++func parseCount(value string) (float64, bool) {+ count, err := strconv.ParseUint(strings.ReplaceAll(value, ",", ""), 10, 64)+ if err != nil {+ return 0, false+ }+ return float64(count), true}diff --git a/backend/plugin/db/cockroachdb/stats_test.go b/backend/plugin/db/cockroachdb/stats_test.gonew file mode 100644index 0000000000..78eea0f223--- /dev/null+++ b/backend/plugin/db/cockroachdb/stats_test.go@@ -0,0 +1,232 @@+package cockroachdb++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "io"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// The fixtures are EXPLAIN outputs captured through this driver from CockroachDB v25.2. After ANALYZE,+// t has 1000 rows with an index on c = id % 100 and none on d = id % 7, s has 100 rows (10 flagged),+// big has 100 rows per s id, big2 has 10000 rows and no secondary index, and parent has 100 rows with+// 10 cascading child rows each. t2 is empty and has no statistics.+func TestGetAffectedRowsFromPlan(t *testing.T) {+ for _, tc := range []struct {+ fixture string+ want int64+ }{+ {fixture: "update_index_join.txt", want: 100},+ // The filter estimates 143 of the 1,000 scanned rows.+ {fixture: "update_filter.txt", want: 143},+ // The limit caps the filter's 143 rows.+ {fixture: "delete_filter_limit.txt", want: 10},+ {fixture: "update_from_lookup_join.txt", want: 1000},+ {fixture: "update_exists_hash_semi_join.txt", want: 1000},+ // The subquery in SET is a sibling of the UPDATE, not its input.+ {fixture: "update_subquery_in_set.txt", want: 100},+ // The anti join against t2 has no estimate, so its input's estimate is used.+ {fixture: "insert_select_on_conflict_do_nothing.txt", want: 100},+ // The foreign key check after the INSERT is not its input.+ {fixture: "insert_select_fk_check.txt", want: 49},+ {fixture: "upsert_select.txt", want: 100},+ {fixture: "upsert_values_select.txt", want: 1},+ {fixture: "insert_fast_path.txt", want: 2},+ // The DELETE returns rows to the CTE, so it has an estimate of its own.+ {fixture: "cte_delete_select.txt", want: 5000},+ // The statement deletes 100 rows and inserts them into t2.+ {fixture: "insert_select_statement_source_delete.txt", want: 200},+ {fixture: "delete_range_fast_path_off.txt", want: 100},+ // The DELETE removes 10 parent rows; the 100 child rows it cascades to are not counted.+ {fixture: "delete_fk_cascade_fast_path_off.txt", want: 10},+ } {+ t.Run(tc.fixture, func(t *testing.T) {+ rows, err := getAffectedRowsFromPlan(readPlanFixture(t, tc.fixture))+ require.NoError(t, err)+ require.Equal(t, tc.want, rows)+ })+ }+}++func TestGetAffectedRowsFromPlanError(t *testing.T) {+ for name, plan := range map[string][]string{+ "select": readPlanFixture(t, "select.txt"),+ "delete range": readPlanFixture(t, "delete_range.txt"),+ "missing stats": readPlanFixture(t, "update_missing_stats.txt"),+ "empty": nil,+ "unparsable estimate": {"• update", "│", "└── • scan", " estimated row count: many"},+ "limit count without an estimate": {"• delete", "│", "└── • limit", " count: 10"},+ } {+ t.Run(name, func(t *testing.T) {+ rows, err := getAffectedRowsFromPlan(plan)+ require.Error(t, err)+ require.Zero(t, rows)+ })+ }+}++func TestCountAffectedRows(t *testing.T) {+ const statement = "DELETE FROM big2 WHERE id <= 100"+ const setLocal = "SET LOCAL optimizer_use_delete_range_fast_path = off"+ for _, tc := range []struct {+ name string+ // input is the statement passed to CountAffectedRows, when it is not statement.+ input string+ plans [][]string+ setLocalErr error+ want int64+ wantErr string+ wantStatements []string+ }{+ {+ name: "search_path_is_set_for_the_explain",+ input: base.WithSearchPath(statement, []string{"app"}),+ plans: [][]string{readPlanFixture(t, "delete_range_fast_path_off.txt")},+ want: 100,+ wantStatements: []string{"BEGIN", `SET LOCAL search_path TO "app"`, "EXPLAIN " + statement, "ROLLBACK"},+ },+ {+ name: "plan_with_an_estimate",+ plans: [][]string{readPlanFixture(t, "delete_range_fast_path_off.txt")},+ want: 100,+ wantStatements: []string{"EXPLAIN " + statement},+ },+ {+ name: "delete_range_is_planned_again_without_the_fast_path",+ plans: [][]string{readPlanFixture(t, "delete_range.txt"), readPlanFixture(t, "delete_range_fast_path_off.txt")},+ want: 100,+ wantStatements: []string{"EXPLAIN " + statement, "BEGIN", setLocal, "EXPLAIN " + statement, "ROLLBACK"},+ },+ {+ name: "version_without_the_setting_keeps_the_delete_range_error",+ plans: [][]string{readPlanFixture(t, "delete_range.txt")},+ setLocalErr: errors.New(`unrecognized configuration parameter "optimizer_use_delete_range_fast_path"`),+ wantErr: "the delete range node in the plan has no row count estimate",+ wantStatements: []string{"EXPLAIN " + statement, "BEGIN", setLocal, "ROLLBACK"},+ },+ {+ name: "plan_without_delete_range_is_not_planned_again",+ plans: [][]string{readPlanFixture(t, "select.txt")},+ wantErr: "the plan has no mutation node",+ wantStatements: []string{"EXPLAIN " + statement},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ connector := &fakeExplainConnector{plans: tc.plans, setLocalErr: tc.setLocalErr}+ db := sql.OpenDB(connector)+ defer db.Close()++ input := tc.input+ if input == "" {+ input = statement+ }+ got, err := (&Driver{db: db}).CountAffectedRows(context.Background(), input)+ if tc.wantErr != "" {+ require.EqualError(t, err, tc.wantErr)+ } else {+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ }+ require.Equal(t, tc.wantStatements, connector.statements)+ })+ }+}++func readPlanFixture(t *testing.T, name string) []string {+ t.Helper()+ content, err := os.ReadFile(filepath.Join("test-data", "explain-plan", name))+ require.NoError(t, err)+ return strings.Split(strings.TrimSuffix(string(content), "\n"), "\n")+}++// fakeExplainConnector answers each EXPLAIN with the next plan and records every statement,+// including transaction control.+type fakeExplainConnector struct {+ plans [][]string+ setLocalErr error+ statements []string+}++func (c *fakeExplainConnector) Connect(context.Context) (driver.Conn, error) {+ return &fakeExplainConn{connector: c}, nil+}++func (*fakeExplainConnector) Driver() driver.Driver {+ return nil+}++type fakeExplainConn struct {+ connector *fakeExplainConnector+}++func (*fakeExplainConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (*fakeExplainConn) Close() error {+ return nil+}++func (c *fakeExplainConn) Begin() (driver.Tx, error) {+ c.connector.statements = append(c.connector.statements, "BEGIN")+ return &fakeExplainTx{connector: c.connector}, nil+}++func (c *fakeExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {+ c.connector.statements = append(c.connector.statements, query)+ return driver.ResultNoRows, c.connector.setLocalErr+}++func (c *fakeExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+ c.connector.statements = append(c.connector.statements, query)+ if len(c.connector.plans) == 0 {+ return nil, errors.New("unexpected EXPLAIN")+ }+ plan := c.connector.plans[0]+ c.connector.plans = c.connector.plans[1:]+ return &fakeExplainRows{lines: plan}, nil+}++type fakeExplainTx struct {+ connector *fakeExplainConnector+}++func (tx *fakeExplainTx) Commit() error {+ tx.connector.statements = append(tx.connector.statements, "COMMIT")+ return nil+}++func (tx *fakeExplainTx) Rollback() error {+ tx.connector.statements = append(tx.connector.statements, "ROLLBACK")+ return nil+}++type fakeExplainRows struct {+ lines []string+}++func (*fakeExplainRows) Columns() []string {+ return []string{"info"}+}++func (*fakeExplainRows) Close() error {+ return nil+}++func (r *fakeExplainRows) Next(dest []driver.Value) error {+ if len(r.lines) == 0 {+ return io.EOF+ }+ dest[0] = r.lines[0]+ r.lines = r.lines[1:]+ return nil+}diff --git a/backend/plugin/db/cockroachdb/sync.go b/backend/plugin/db/cockroachdb/sync.goindex 9888136eef..4140c35286 100644--- a/backend/plugin/db/cockroachdb/sync.go+++ b/backend/plugin/db/cockroachdb/sync.go@@ -78,6 +78,9 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeif databaseMetadata == nil {return nil, common.Errorf(common.NotFound, "database %q not found", d.databaseName)}+ if err := d.db.QueryRowContext(ctx, "SHOW search_path").Scan(&databaseMetadata.SearchPath); err != nil {+ return nil, errors.Wrapf(err, "failed to get search path for database %q", d.databaseName)+ }isAtLeastPG10 := isAtLeastPG10(d.connectionCtx.EngineVersion)txn, err := d.db.BeginTx(ctx, nil)diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txtnew file mode 100644index 0000000000..fd5f021289--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txt@@ -0,0 +1,29 @@+distribution: local+vectorized: true++• root+│+├── • group (scalar)+│ │ estimated row count: 1+│ │+│ └── • scan buffer+│ estimated row count: 5,000+│ label: buffer 1 (d)+│+└── • subquery+ │ id: @S1+ │ original sql: DELETE FROM big WHERE s_id <= 50 RETURNING id+ │ exec mode: discard all rows+ │+ └── • buffer+ │ label: buffer 1 (d)+ │+ └── • delete+ │ estimated row count: 5,000+ │ from: big+ │+ └── • scan+ estimated row count: 5,000 (50% of the table; stats collected 40 seconds ago)+ table: big@idx_s+ spans: [ - /50]+ locking strength: for updatediff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txtnew file mode 100644index 0000000000..edcca5af10--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txt@@ -0,0 +1,22 @@+distribution: local+vectorized: true++• delete+│ from: t+│ auto commit+│+└── • limit+ │ count: 10+ │+ └── • filter+ │ estimated row count: 143+ │ filter: d = 3+ │+ └── • scan+ estimated row count: 70 - 1,000 (100% of the table; stats collected 40 seconds ago)+ table: t@t_pkey+ spans: FULL SCAN (SOFT LIMIT)++index recommendations: 1+1. type: index creation+ SQL command: CREATE INDEX ON capture.public.t (d) STORING (c);diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txtnew file mode 100644index 0000000000..90bad021ff--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txt@@ -0,0 +1,28 @@+distribution: local+vectorized: true++• root+│+├── • delete+│ │ from: parent+│ │+│ └── • scan+│ estimated row count: 10 (10% of the table; stats collected 39 seconds ago)+│ table: parent@parent_pkey+│ spans: [ - /10]+│ locking strength: for update+│+└── • fk-cascade+ │ fk: child_parent_id_fkey+ │+ └── • delete+ │ from: child+ │+ └── • filter+ │ estimated row count: 100+ │ filter: parent_id <= 10+ │+ └── • scan+ estimated row count: 1,000 (100% of the table; stats collected 39 seconds ago)+ table: child@child_pkey+ spans: FULL SCANdiff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txtnew file mode 100644index 0000000000..2d2382b49c--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txt@@ -0,0 +1,6 @@+distribution: local+vectorized: true++• delete range+ from: big2+ spans: [ - /100]diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txtnew file mode 100644index 0000000000..f6c7baf4ca--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txt@@ -0,0 +1,11 @@+distribution: local+vectorized: true++• delete+│ from: big2+│+└── • scan+ estimated row count: 100 (1.0% of the table; stats collected 40 seconds ago)+ table: big2@big2_pkey+ spans: [ - /100]+ locking strength: for updatediff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txtnew file mode 100644index 0000000000..15cd23e67b--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txt@@ -0,0 +1,7 @@+distribution: local+vectorized: true++• insert fast path+ into: t2(id, c, d, grp, v)+ auto commit+ size: 5 columns, 2 rowsdiff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txtnew file mode 100644index 0000000000..ab8c249c1e--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txt@@ -0,0 +1,31 @@+distribution: local+vectorized: true++• root+│+├── • insert+│ │ into: child(id, parent_id, v)+│ │+│ └── • buffer+│ │ label: buffer 1+│ │+│ └── • render+│ │+│ └── • scan+│ estimated row count: 49 (49% of the table; stats collected 39 seconds ago)+│ table: parent@parent_pkey+│ spans: [ - /49]+│+└── • constraint-check+ │+ └── • error if rows+ │+ └── • lookup join (anti)+ │ estimated row count: 0+ │ table: parent@parent_pkey+ │ equality: (id) = (id)+ │ equality cols are key+ │+ └── • scan buffer+ estimated row count: 49+ label: buffer 1diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txtnew file mode 100644index 0000000000..e2de9b7396--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txt@@ -0,0 +1,25 @@+distribution: local+vectorized: true++• insert+│ into: t2(id, c, d, grp, v)+│ auto commit+│ arbiter indexes: t2_pkey+│+└── • lookup join (anti)+ │ table: t2@t2_pkey+ │ equality: (id) = (id)+ │ equality cols are key+ │+ └── • index join+ │ estimated row count: 100+ │ table: t@t_pkey+ │+ └── • scan+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+ table: t@idx_c+ spans: [ - /9]++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txtnew file mode 100644index 0000000000..6850426752--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txt@@ -0,0 +1,38 @@+distribution: local+vectorized: true++• root+│+├── • insert+│ │ into: t2(id, c, d, grp, v)+│ │+│ └── • scan buffer+│ estimated row count: 100+│ label: buffer 1+│+└── • subquery+ │ id: @S1+ │ original sql: DELETE FROM t WHERE c < 10 RETURNING *+ │ exec mode: discard all rows+ │+ └── • buffer+ │ label: buffer 1+ │+ └── • delete+ │ estimated row count: 100+ │ from: t+ │+ └── • index join+ │ estimated row count: 100+ │ table: t@t_pkey+ │ locking strength: for update+ │+ └── • scan+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+ table: t@idx_c+ spans: [ - /9]+ locking strength: for update++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txtnew file mode 100644index 0000000000..a9f68cb42c--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txt@@ -0,0 +1,15 @@+distribution: local+vectorized: true++• index join+│ estimated row count: 100+│ table: t@t_pkey+│+└── • scan+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+ table: t@idx_c+ spans: [ - /9]++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txtnew file mode 100644index 0000000000..17cff77bea--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txt@@ -0,0 +1,32 @@+distribution: local+vectorized: true++• update+│ table: t+│ set: v+│ auto commit+│+└── • render+ │+ └── • hash join (semi)+ │ estimated row count: 1,000+ │ equality: (grp) = (id)+ │ right cols are key+ │+ ├── • scan+ │ estimated row count: 1,000 (100% of the table; stats collected 40 seconds ago)+ │ table: t@t_pkey+ │ spans: FULL SCAN+ │+ └── • filter+ │ estimated row count: 10+ │ filter: flag+ │+ └── • scan+ estimated row count: 100 (100% of the table; stats collected 40 seconds ago)+ table: s@s_pkey+ spans: FULL SCAN++index recommendations: 1+1. type: index creation+ SQL command: CREATE INDEX ON capture.public.t (grp) STORING (c, d, v);diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txtnew file mode 100644index 0000000000..f2dbdd922c--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txt@@ -0,0 +1,22 @@+distribution: local+vectorized: true++• update+│ table: t+│ set: v+│ auto commit+│+└── • render+ │+ └── • filter+ │ estimated row count: 143+ │ filter: d = 3+ │+ └── • scan+ estimated row count: 1,000 (100% of the table; stats collected 40 seconds ago)+ table: t@t_pkey+ spans: FULL SCAN++index recommendations: 1+1. type: index creation+ SQL command: CREATE INDEX ON capture.public.t (d) STORING (c, grp, v);diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txtnew file mode 100644index 0000000000..d5735708b5--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txt@@ -0,0 +1,33 @@+distribution: local+vectorized: true++• update+│ table: big+│ set: v+│ auto commit+│+└── • render+ │+ └── • lookup join+ │ estimated row count: 1,000+ │ table: big@big_pkey+ │ equality: (id) = (id)+ │ equality cols are key+ │+ └── • lookup join+ │ estimated row count: 1,000+ │ table: big@idx_s+ │ equality: (id) = (s_id)+ │+ └── • filter+ │ estimated row count: 10+ │ filter: flag+ │+ └── • scan+ estimated row count: 100 (100% of the table; stats collected 40 seconds ago)+ table: s@s_pkey+ spans: FULL SCAN++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.big (s_id) STORING (v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.big@idx_s NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.big@idx_s` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txtnew file mode 100644index 0000000000..3859871c15--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txt@@ -0,0 +1,24 @@+distribution: local+vectorized: true++• update+│ table: t+│ set: v+│ auto commit+│+└── • render+ │+ └── • index join+ │ estimated row count: 100+ │ table: t@t_pkey+ │ locking strength: for update+ │+ └── • scan+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+ table: t@idx_c+ spans: [ - /9]+ locking strength: for update++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txtnew file mode 100644index 0000000000..0ce7fcbc26--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txt@@ -0,0 +1,21 @@+distribution: local+vectorized: true++• update+│ table: t2+│ set: v+│ auto commit+│+└── • render+ │+ └── • filter+ │ filter: c = 1+ │+ └── • scan+ missing stats+ table: t2@t2_pkey+ spans: FULL SCAN++index recommendations: 1+1. type: index creation+ SQL command: CREATE INDEX ON capture.public.t2 (c) STORING (d, grp, v);diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txtnew file mode 100644index 0000000000..1a7937fc42--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txt@@ -0,0 +1,45 @@+distribution: local+vectorized: true++• root+│+├── • update+│ │ table: t+│ │ set: v+│ │ auto commit+│ │+│ └── • render+│ │+│ └── • index join+│ │ estimated row count: 100+│ │ table: t@t_pkey+│ │ locking strength: for update+│ │+│ └── • scan+│ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+│ table: t@idx_c+│ spans: [ - /9]+│ locking strength: for update+│+└── • subquery+ │ id: @S1+ │ original sql: (SELECT v FROM s JOIN t ON t.id = s.id LIMIT 1)+ │ exec mode: one row+ │+ └── • limit+ │ count: 1+ │+ └── • lookup join+ │ estimated row count: 100+ │ table: s@s_pkey+ │ equality: (id) = (id)+ │ equality cols are key+ │+ └── • scan+ estimated row count: 100 - 1,000 (100% of the table; stats collected 40 seconds ago)+ table: t@t_pkey+ spans: FULL SCAN (SOFT LIMIT)++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txtnew file mode 100644index 0000000000..c8a528438e--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txt@@ -0,0 +1,19 @@+distribution: local+vectorized: true++• upsert+│ into: t2(id, c, d, grp, v)+│ auto commit+│+└── • index join+ │ estimated row count: 100+ │ table: t@t_pkey+ │+ └── • scan+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)+ table: t@idx_c+ spans: [ - /9]++index recommendations: 1+1. type: index replacement+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txtnew file mode 100644index 0000000000..235e3b5ad9--- /dev/null+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txt@@ -0,0 +1,9 @@+distribution: local+vectorized: true++• upsert+│ into: t2(id, c, d, grp, v)+│ auto commit+│+└── • values+ size: 5 columns, 1 rowdiff --git a/backend/plugin/db/mssql/stats.go b/backend/plugin/db/mssql/stats.goindex bfe1653c07..8faf4b7ed1 100644--- a/backend/plugin/db/mssql/stats.go+++ b/backend/plugin/db/mssql/stats.go@@ -3,8 +3,8 @@ package mssqlimport ("context""database/sql"- "math"+ "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/plugin/db/util")@@ -47,7 +47,7 @@ func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64}if rowsColumn.Valid {- return int64(math.Round(rowsColumn.Float64)), nil+ return common.RoundRows(rowsColumn.Float64), nil}}if err := rows.Err(); err != nil {diff --git a/backend/plugin/db/mssql/stats_test.go b/backend/plugin/db/mssql/stats_test.goindex 8f695eb5f5..35c287e7cc 100644--- a/backend/plugin/db/mssql/stats_test.go+++ b/backend/plugin/db/mssql/stats_test.go@@ -32,10 +32,27 @@ GOINSERT INTO dbo.t (c1) VALUES (1),(2),(3),(4),(5);GO+CREATE TABLE dbo.s (id INT PRIMARY KEY, flag BIT NOT NULL);+CREATE TABLE dbo.big (id INT PRIMARY KEY, s_id INT NOT NULL, v INT NOT NULL);+CREATE TABLE dbo.big_archive (id INT PRIMARY KEY, s_id INT NOT NULL, v INT NOT NULL);+GO++-- s flags 10 of its 100 rows and big holds 100 rows for each row of s, so+-- joining big to the flagged rows matches 1000 of its 10000 rows.+INSERT INTO dbo.s SELECT n, CASE WHEN n <= 10 THEN 1 ELSE 0 END FROM (SELECT TOP (100) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n FROM sys.all_objects) AS seq;+INSERT INTO dbo.big SELECT n, n % 100 + 1, 0 FROM (SELECT TOP (10000) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n FROM sys.all_objects AS a CROSS JOIN sys.all_objects AS b) AS seq;+GO++CREATE INDEX big_s_id ON dbo.big (s_id);+CREATE STATISTICS s_flag ON dbo.s (flag);+GO+-- EstimateRows is a cardinality estimate, so it is only stable once the-- optimizer has statistics. Without them SQL Server falls back to guessing a-- fixed selectivity for a range predicate and the counts below drift.UPDATE STATISTICS dbo.t WITH FULLSCAN;+UPDATE STATISTICS dbo.s WITH FULLSCAN;+UPDATE STATISTICS dbo.big WITH FULLSCAN;GO`)@@ -68,6 +85,63 @@ GOrequire.EqualValues(t, 3, rows)})+ for _, tc := range []struct {+ name string+ statement string+ // changedRows is how many rows the statement would change. An estimate can miss it,+ // as MERGE estimates 899 of 1000 on SQL Server 2022, so the bounds allow a factor of+ // two, which still excludes the 10, 100, and 10000 rows that the join inputs estimate.+ changedRows int64+ }{+ {+ name: "delete_with_predicate",+ statement: `DELETE FROM dbo.big WHERE s_id <= 3;`,+ changedRows: 300,+ },+ {+ name: "delete_join",+ statement: `DELETE b FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,+ changedRows: 1000,+ },+ {+ name: "update_from_join",+ statement: `UPDATE b SET v = 1 FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,+ changedRows: 1000,+ },+ {+ name: "delete_where_in_subquery",+ statement: `DELETE FROM dbo.big WHERE s_id IN (SELECT id FROM dbo.s WHERE flag = 1);`,+ changedRows: 1000,+ },+ {+ name: "insert_select_with_filter",+ statement: `INSERT INTO dbo.big_archive SELECT id, s_id, v FROM dbo.big WHERE s_id > 95;`,+ changedRows: 500,+ },+ {+ name: "insert_select_with_join",+ statement: `INSERT INTO dbo.big_archive SELECT b.id, b.s_id, b.v FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,+ changedRows: 1000,+ },+ {+ name: "merge",+ statement: `MERGE dbo.big_archive AS tgt+USING (SELECT b.id, b.s_id, b.v FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1) AS src+ON tgt.id = src.id+WHEN NOT MATCHED THEN INSERT (id, s_id, v) VALUES (src.id, src.s_id, src.v);`,+ changedRows: 1000,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()++ rows, err := driver.CountAffectedRows(ctx, tc.statement)+ require.NoError(t, err)+ require.Greater(t, rows, tc.changedRows/2)+ require.LessOrEqual(t, rows, tc.changedRows*2)+ })+ }+// The seam between the two halves: neither the tsql extractor tests nor the// subtests above can catch a sample that the extractor emits but SHOWPLAN// rejects, so feed the extractor's own output in rather than a literal.@@ -82,9 +156,9 @@ GOsummary, err := parserbase.ExtractChangedResources(storepb.Engine_MSSQL, "resource_change_test", "dbo", nil, parserbase.ExtractASTs(stmts), statement)require.NoError(t, err)- require.Len(t, summary.SampleDMLS, 1)+ require.Len(t, summary.DMLStatements, 1)- sampled, err := driver.CountAffectedRows(ctx, summary.SampleDMLS[0])+ sampled, err := driver.CountAffectedRows(ctx, summary.DMLStatements[0])require.NoError(t, err, "SHOWPLAN must accept the sample the extractor emits")want, err := driver.CountAffectedRows(ctx, bare)require.NoError(t, err)diff --git a/backend/plugin/db/mysql/stats.go b/backend/plugin/db/mysql/stats.goindex afe6db5da7..6835dca530 100644--- a/backend/plugin/db/mysql/stats.go+++ b/backend/plugin/db/mysql/stats.go@@ -3,221 +3,165 @@ package mysqlimport ("context""database/sql"- "encoding/json"- "fmt"+ "log/slog"+ "slices"+ "strconv""strings""github.com/bytebase/omni/mysql/ast"+ "github.com/go-sql-driver/mysql""github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/db/util"mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")+// erUnknownSystemVariable is the MySQL error number for setting a variable the server lacks.+const erUnknownSystemVariable = 1193++// CountAffectedRows returns the planner's estimate of the rows the INSERT, UPDATE, or DELETE+// statement modifies.func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {if d.dbType == storepb.Engine_OCEANBASE {- return countAffectedRowsForOceanBase(ctx, d.db, statement)- }-- // Prefer the JSON query plan: the tabular EXPLAIN's first "rows" value may be the- // scan estimate of a driving table unrelated to the DML target (BYT-9858). Fall- // back to the tabular heuristic when no flagged target node is found (e.g. INSERT- // plans, or servers that don't mark the target).- if count, ok := d.countAffectedRowsFromJSONPlan(ctx, statement); ok {- return capAffectedRowsByLimit(count, statement), nil+ rows, err := d.db.QueryContext(ctx, "EXPLAIN FORMAT=JSON "+statement)+ if err != nil {+ return 0, err+ }+ defer rows.Close()+ plan, err := readExplainJSON(rows)+ if err != nil {+ return 0, err+ }+ return mysqlparser.EstimateAffectedRowsFromOceanBaseExplainJSON(plan)}- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)- rows, err := d.db.QueryContext(ctx, explainSQL)+ list, err := mysqlparser.ParseMySQL(statement)if err != nil {- return 0, err+ return 0, errors.Wrap(err, "failed to parse statement")}- defer rows.Close()-- // mysql> explain delete from td;- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- // | 1 | DELETE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | NULL |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- //- // mysql> explain insert into td select * from td;- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+- // | 1 | INSERT | td | NULL | ALL | NULL | NULL | NULL | NULL | NULL | NULL | NULL |- // | 1 | SIMPLE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | Using temporary |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+- columns, err := rows.Columns()+ if list == nil || len(list.Items) != 1 {+ return 0, errors.New("expected exactly one statement")+ }+ query := mysqlparser.AffectedRowsQuery(list.Items[0], statement)+ plan, err := ExplainJSON(ctx, d.db, query)if err != nil {return 0, err}- rowsIndex, ok := util.GetColumnIndex(columns, "rows")- if !ok {- return 0, nil- }- for rows.Next() {- scanArgs := make([]any, len(columns))- for i := range scanArgs {- var unused any- scanArgs[i] = &unused- }- var rowsColumn sql.NullInt64- scanArgs[rowsIndex] = &rowsColumn- if err := rows.Scan(scanArgs...); err != nil {- return 0, err- }+ return EstimateAffectedRows(ctx, d.db, list.Items[0], query, plan)+}- if rowsColumn.Valid {- return capAffectedRowsByLimit(rowsColumn.Int64, statement), nil- }+// EstimateAffectedRows returns the rows stmt modifies according to plan, the ExplainJSON output for+// query, which is the statement's AffectedRowsQuery. When plan has no estimate to read, as MySQL 8.0+// prints no source plan for an INSERT ... SELECT that computes window functions, the estimate of the+// tabular EXPLAIN stands in for it.+func EstimateAffectedRows(ctx context.Context, db *sql.DB, stmt ast.Node, query string, plan string) (int64, error) {+ count, err := mysqlparser.EstimateAffectedRowsFromExplainJSON(stmt, plan)+ if err == nil {+ return count, nil}- if err := rows.Err(); err != nil {+ rows, tables, ok := explainTabularEstimate(ctx, db, query)+ if !ok {return 0, err}- return 0, nil+ // A multi-table UPDATE or DELETE can change a row of each target for every row its join produces.+ return mysqlparser.CapAffectedRowsByLimit(stmt, rows*float64(mysqlparser.DMLTargetCount(stmt, tables))), nil}-func (d *Driver) countAffectedRowsFromJSONPlan(ctx context.Context, statement string) (int64, bool) {- rows, err := d.db.QueryContext(ctx, fmt.Sprintf("EXPLAIN FORMAT=JSON %s", statement))+// explainTabularEstimate returns the rows that the first query block of the tabular EXPLAIN of query+// produces, and how many tables it joins: the product, over those tables, of the rows each reads+// scaled by its filtered percentage where the server prints one.+func explainTabularEstimate(ctx context.Context, db *sql.DB, query string) (float64, int, bool) {+ // MySQL 9 prints a plain EXPLAIN in the TREE format.+ rows, err := db.QueryContext(ctx, "EXPLAIN FORMAT=TRADITIONAL "+query)if err != nil {- return 0, false+ return 0, 0, false}defer rows.Close()-- var plan strings.Builder- for rows.Next() {- var planColumn sql.NullString- if err := rows.Scan(&planColumn); err != nil {- return 0, false- }- if planColumn.Valid {- plan.WriteString(planColumn.String)- }- }- if err := rows.Err(); err != nil {- return 0, false- }- return mysqlparser.GetEstimatedAffectedRowsFromExplainJSON(plan.String())-}--func capAffectedRowsByLimit(count int64, statement string) int64 {- limit, ok := affectedRowsLimit(statement)- if !ok || limit >= count {- return count- }- return limit-}--func affectedRowsLimit(statement string) (int64, bool) {- list, err := mysqlparser.ParseMySQL(statement)- if err != nil || list == nil || len(list.Items) != 1 {- return 0, false- }-- var limit *ast.Limit- switch stmt := list.Items[0].(type) {- case *ast.UpdateStmt:- limit = stmt.Limit- case *ast.DeleteStmt:- limit = stmt.Limit- case *ast.InsertStmt:- if stmt.Select != nil {- limit = stmt.Select.Limit- }- default:- return 0, false+ columns, err := rows.Columns()+ if err != nil {+ return 0, 0, false}- if limit == nil || limit.Count == nil {- return 0, false+ column := func(name string) int {+ return slices.IndexFunc(columns, func(column string) bool { return strings.EqualFold(column, name) })}- lit, ok := limit.Count.(*ast.IntLit)- if !ok || lit.Value < 0 {- return 0, false+ idIndex, rowsIndex, filteredIndex := column("id"), column("rows"), column("filtered")+ if idIndex < 0 || rowsIndex < 0 {+ return 0, 0, false}- return lit.Value, true-}--func countAffectedRowsForOceanBase(ctx context.Context, sqlDB *sql.DB, dml string) (int64, error) {- explainSQL := fmt.Sprintf("EXPLAIN FORMAT=JSON %s", dml)- rows, err := sqlDB.QueryContext(ctx, explainSQL)- if err != nil {- return 0, err+ values := make([]sql.NullString, len(columns))+ dest := make([]any, len(columns))+ for i := range values {+ dest[i] = &values[i]}- defer rows.Close()- var planString strings.Builder+ var blockID string+ estimate, tables := 1.0, 0for rows.Next() {- var planColumn sql.NullString- if err := rows.Scan(&planColumn); err != nil {- return 0, err+ if err := rows.Scan(dest...); err != nil {+ return 0, 0, false}- if !planColumn.Valid {+ tableRows, err := strconv.ParseFloat(values[rowsIndex].String, 64)+ if !values[rowsIndex].Valid || err != nil || (tables > 0 && values[idIndex].String != blockID) {continue}- planString.WriteString(planColumn.String)- }- if err := rows.Err(); err != nil {- return 0, err- }- if planString.Len() == 0 {- return 0, nil+ blockID = values[idIndex].String+ tables+++ estimate *= tableRows+ if filteredIndex >= 0 && values[filteredIndex].Valid {+ if filtered, err := strconv.ParseFloat(values[filteredIndex].String, 64); err == nil {+ estimate *= filtered / 100+ }+ }}- return getAffectedRowsFromOceanBaseQueryPlan(planString.String())+ return estimate, tables, tables > 0 && rows.Err() == nil}-func getAffectedRowsFromOceanBaseQueryPlan(planString string) (int64, error) {- var planValue map[string]json.RawMessage- if err := json.Unmarshal([]byte(planString), &planValue); err != nil {- return 0, errors.Wrapf(err, "failed to parse query plan from string: %+v", planString)- }- if len(planValue) == 0 {- return 0, nil- }- queryPlan := oceanBaseQueryPlan{}- if err := queryPlan.Unmarshal(planValue); err != nil {- return 0, errors.Wrapf(err, "failed to parse query plan from map: %+v", planValue)- }- if queryPlan.Operator != "" {- return queryPlan.EstRows, nil+// ExplainJSON returns the `EXPLAIN FORMAT=JSON` output for the statement in the JSON format+// version that mysqlparser.EstimateAffectedRowsFromExplainJSON reads.+func ExplainJSON(ctx context.Context, db *sql.DB, statement string) (string, error) {+ // The session variable must be set on the connection that runs the EXPLAIN.+ conn, err := db.Conn(ctx)+ if err != nil {+ return "", err}- count := int64(-1)- for k, v := range planValue {- if !strings.HasPrefix(k, "CHILD_") {- continue- }- child := oceanBaseQueryPlan{}- if err := child.Unmarshal(v); err != nil {- return 0, errors.Wrapf(err, "failed to parse field '%s', value: %+v", k, v)- }- if child.Operator != "" && child.EstRows > count {- count = child.EstRows+ defer conn.Close()++ // MySQL 9.5 defaults explain_json_format_version to 2. MySQL before 8.3 and MariaDB lack the+ // variable and only produce version 1.+ if _, err := conn.ExecContext(ctx, "SET SESSION explain_json_format_version = 1"); err != nil {+ var mysqlErr *mysql.MySQLError+ if !errors.As(err, &mysqlErr) || mysqlErr.Number != erUnknownSystemVariable {+ return "", errors.Wrap(err, "failed to set explain_json_format_version")}+ } else {+ // The connection returns to the pool, so later statements on it get the server's version back.+ defer func() {+ if _, err := conn.ExecContext(ctx, "SET SESSION explain_json_format_version = DEFAULT"); err != nil {+ slog.Warn("failed to reset explain_json_format_version", log.BBError(err))+ }+ }()}- if count >= 0 {- return count, nil- }- return 0, nil-}-// oceanBaseQueryPlan represents the query plan of OceanBase.-type oceanBaseQueryPlan struct {- ID int `json:"ID"`- Operator string `json:"OPERATOR"`- Name string `json:"NAME"`- EstRows int64 `json:"EST.ROWS"`- Cost int `json:"COST"`- OutPut any `json:"output"`+ rows, err := conn.QueryContext(ctx, "EXPLAIN FORMAT=JSON "+statement)+ if err != nil {+ return "", err+ }+ defer rows.Close()+ return readExplainJSON(rows)}-// Unmarshal parses data and stores the result to current oceanBaseQueryPlan.-func (plan *oceanBaseQueryPlan) Unmarshal(data any) error {- b, err := json.Marshal(data)- if err != nil {- return err+// readExplainJSON concatenates the plan rows; OceanBase splits its JSON plan across rows.+func readExplainJSON(rows *sql.Rows) (string, error) {+ var plan strings.Builder+ for rows.Next() {+ var line sql.NullString+ if err := rows.Scan(&line); err != nil {+ return "", err+ }+ plan.WriteString(line.String)}- if b != nil {- return json.Unmarshal(b, &plan)+ if err := rows.Err(); err != nil {+ return "", err}- return nil+ return plan.String(), nil}diff --git a/backend/plugin/db/mysql/stats_test.go b/backend/plugin/db/mysql/stats_test.goindex 0ca99486b6..f13615ba78 100644--- a/backend/plugin/db/mysql/stats_test.go+++ b/backend/plugin/db/mysql/stats_test.go@@ -8,15 +8,32 @@ import ("strings""testing"+ "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors""github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store")var testOceanBaseExplainRows [][]driver.Value-var testMySQLExplainRows [][]driver.Value-// testMySQLExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries; when empty, the-// JSON query returns no rows and CountAffectedRows falls back to the tabular rows.-var testMySQLExplainJSON string+var (+ // testMySQLExplainJSON is the plan the fake MySQL server returns for EXPLAIN FORMAT=JSON.+ testMySQLExplainJSON string+ // testMySQLExplainTable is the plan the fake MySQL server returns for a tabular EXPLAIN, with the+ // columns in testMySQLExplainTableColumns.+ testMySQLExplainTable [][]driver.Value+ // testMySQLSetErr is the error the fake MySQL server returns for SET statements.+ testMySQLSetErr error+ // testMySQLStatements records every statement the fake MySQL server receives, in order.+ testMySQLStatements []testMySQLStatement+ testMySQLConnections int+)++type testMySQLStatement struct {+ conn int+ query string+}func init() {sql.Register("test_oceanbase_explain", testOceanBaseExplainDriver{})@@ -44,80 +61,76 @@ func (testOceanBaseExplainConn) Begin() (driver.Tx, error) {}func (testOceanBaseExplainConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {- return &testOceanBaseExplainResultRows{rows: testOceanBaseExplainRows}, nil-}--type testOceanBaseExplainResultRows struct {- rows [][]driver.Value- idx int-}--func (*testOceanBaseExplainResultRows) Columns() []string {- return []string{"Query Plan"}-}--func (*testOceanBaseExplainResultRows) Close() error {- return nil-}--func (r *testOceanBaseExplainResultRows) Next(dest []driver.Value) error {- if r.idx >= len(r.rows) {- return io.EOF- }- copy(dest, r.rows[r.idx])- r.idx++- return nil+ return &testExplainResultRows{rows: testOceanBaseExplainRows}, nil}type testMySQLExplainDriver struct{}func (testMySQLExplainDriver) Open(string) (driver.Conn, error) {- return testMySQLExplainConn{}, nil+ testMySQLConnections+++ return &testMySQLExplainConn{id: testMySQLConnections}, nil}-type testMySQLExplainConn struct{}+type testMySQLExplainConn struct {+ id int+}-func (testMySQLExplainConn) Prepare(string) (driver.Stmt, error) {+func (*testMySQLExplainConn) Prepare(string) (driver.Stmt, error) {return nil, driver.ErrSkip}-func (testMySQLExplainConn) Close() error {+func (*testMySQLExplainConn) Close() error {return nil}-func (testMySQLExplainConn) Begin() (driver.Tx, error) {+func (*testMySQLExplainConn) Begin() (driver.Tx, error) {return nil, driver.ErrSkip}-func (testMySQLExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {- if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON") {- var rows [][]driver.Value- if testMySQLExplainJSON != "" {- rows = [][]driver.Value{{testMySQLExplainJSON}}- }- return &testMySQLExplainResultRows{columns: []string{"EXPLAIN"}, rows: rows}, nil+func (c *testMySQLExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {+ testMySQLStatements = append(testMySQLStatements, testMySQLStatement{conn: c.id, query: query})+ if !strings.HasPrefix(query, "SET ") {+ return nil, errors.Errorf("unexpected exec %q", query)+ }+ if testMySQLSetErr != nil {+ return nil, testMySQLSetErr+ }+ return driver.ResultNoRows, nil+}++var testMySQLExplainTableColumns = []string{"id", "select_type", "table", "rows", "filtered"}++func (c *testMySQLExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+ testMySQLStatements = append(testMySQLStatements, testMySQLStatement{conn: c.id, query: query})+ switch {+ case strings.HasPrefix(query, "EXPLAIN FORMAT=JSON "):+ return &testExplainResultRows{rows: [][]driver.Value{{testMySQLExplainJSON}}}, nil+ case strings.HasPrefix(query, "EXPLAIN "):+ return &testExplainResultRows{columns: testMySQLExplainTableColumns, rows: testMySQLExplainTable}, nil+ default:+ return nil, errors.Errorf("unexpected query %q", query)}- return &testMySQLExplainResultRows{- columns: []string{"id", "select_type", "table", "type", "rows", "filtered"},- rows: testMySQLExplainRows,- }, nil}-type testMySQLExplainResultRows struct {+type testExplainResultRows struct {+ // columns defaults to the single EXPLAIN column of a JSON plan.columns []stringrows [][]driver.Valueidx int}-func (r *testMySQLExplainResultRows) Columns() []string {- return r.columns+func (r *testExplainResultRows) Columns() []string {+ if r.columns != nil {+ return r.columns+ }+ return []string{"EXPLAIN"}}-func (*testMySQLExplainResultRows) Close() error {+func (*testExplainResultRows) Close() error {return nil}-func (r *testMySQLExplainResultRows) Next(dest []driver.Value) error {+func (r *testExplainResultRows) Next(dest []driver.Value) error {if r.idx >= len(r.rows) {return io.EOF}@@ -126,74 +139,220 @@ func (r *testMySQLExplainResultRows) Next(dest []driver.Value) error {return nil}-func TestCountAffectedRowsCapsExplainEstimateByLimit(t *testing.T) {- testMySQLExplainJSON = ""- testMySQLExplainRows = [][]driver.Value{- {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},- }+// newTestMySQLDriver returns a driver over the fake MySQL server whose pool keeps no idle+// connections, so statements that do not share a pinned connection land on different ones.+func newTestMySQLDriver(t *testing.T, plan string) *Driver {+ t.Helper()+ testMySQLExplainJSON = plan+ testMySQLExplainTable = nil+ testMySQLSetErr = nil+ testMySQLStatements = nildb, err := sql.Open("test_mysql_explain", "")require.NoError(t, err)- defer db.Close()+ db.SetMaxIdleConns(0)+ t.Cleanup(func() { _ = db.Close() })+ return &Driver{dbType: storepb.Engine_MYSQL, db: db}+}- driver := &Driver{db: db}+const (+ // The flagged UPDATE target is scanned first; the join filter on s comes after it.+ testUpdateTargetFirstPlan = `{"query_block":{"select_id":1,"nested_loop":[+ {"table":{"update":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00"}},+ {"table":{"table_name":"s","access_type":"eq_ref","rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00"}}+ ]}}`+ testInsertSelectPlan = `{"query_block":{"select_id":1,+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}+ }}`+)+func TestCountAffectedRowsEstimatesFromJSONPlan(t *testing.T) {+ // A single-table plan of the SELECT a single-table UPDATE or DELETE is explained as.+ const filteredScanPlan = `{"query_block":{"select_id":1,"table":{"table_name":"td","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}}`for _, tc := range []struct {+ name string+ plan stringstatement string+ explained stringwant int64}{{- statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 10;",+ name: "update bounded by the join after the target",+ plan: testUpdateTargetFirstPlan,+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",+ explained: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",+ want: 1133,+ },+ {+ name: "single-table update explained as a select",+ plan: filteredScanPlan,+ statement: "UPDATE td SET c = 1 WHERE d = 0;",+ explained: "SELECT 1 FROM td WHERE d = 0",+ want: 100,+ },+ {+ name: "update capped by LIMIT",+ plan: filteredScanPlan,+ statement: "UPDATE td SET c = 1 WHERE d = 0 LIMIT 10;",+ explained: "SELECT 1 FROM td WHERE d = 0 LIMIT 10",want: 10,},{- statement: "DELETE FROM td WHERE c = 0 LIMIT 20;",+ name: "delete capped by LIMIT",+ plan: filteredScanPlan,+ statement: "DELETE FROM td WHERE d = 0 ORDER BY id LIMIT 20;",+ explained: "SELECT 1 FROM td WHERE d = 0 ORDER BY id LIMIT 20",want: 20,},{- statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 30;",- want: 30,+ name: "insert select reads the source plan",+ plan: testInsertSelectPlan,+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ explained: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ want: 100,},{- statement: "UPDATE td SET c = 1 WHERE c = 0;",- want: 1000,+ name: "insert select capped by LIMIT",+ plan: testInsertSelectPlan,+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 30;",+ explained: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 30;",+ want: 30,},} {- got, err := driver.CountAffectedRows(context.Background(), tc.statement)- require.NoError(t, err)- require.Equal(t, tc.want, got, tc.statement)+ t.Run(tc.name, func(t *testing.T) {+ d := newTestMySQLDriver(t, tc.plan)+ got, err := d.CountAffectedRows(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ require.Equal(t, "EXPLAIN FORMAT=JSON "+tc.explained, testMySQLStatements[1].query)+ })}}-func TestCountAffectedRowsPrefersJSONPlanTargetNode(t *testing.T) {- // The tabular EXPLAIN's first row is a driving-table scan estimate; the JSON plan- // flags the update target whose cumulative estimate is far smaller (BYT-9858).- testMySQLExplainJSON = `{"query_block":{"select_id":1,"nested_loop":[- {"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":100232,"rows_produced_per_join":3006,"filtered":"3.00"}},- {"table":{"update":true,"table_name":"o","access_type":"ref","rows_examined_per_scan":3,"rows_produced_per_join":1144,"filtered":"10.00"}}- ]}}`- testMySQLExplainRows = [][]driver.Value{- {int64(1), "SIMPLE", "t", "ALL", int64(100232), "3.00"},- {int64(1), "UPDATE", "o", "ref", int64(3), "10.00"},+func TestCountAffectedRowsScopesJSONFormatVersionToTheExplain(t *testing.T) {+ const statement = "INSERT INTO t2 SELECT * FROM t WHERE d = 3;"+ for _, tc := range []struct {+ name string+ setErr error+ wantStatements []string+ }{+ {+ name: "server with explain_json_format_version",+ wantStatements: []string{+ "SET SESSION explain_json_format_version = 1",+ "EXPLAIN FORMAT=JSON " + statement,+ "SET SESSION explain_json_format_version = DEFAULT",+ },+ },+ {+ name: "server without explain_json_format_version",+ setErr: &mysql.MySQLError{Number: 1193, Message: "Unknown system variable 'explain_json_format_version'"},+ wantStatements: []string{+ "SET SESSION explain_json_format_version = 1",+ "EXPLAIN FORMAT=JSON " + statement,+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ d := newTestMySQLDriver(t, testInsertSelectPlan)+ testMySQLSetErr = tc.setErr++ got, err := d.CountAffectedRows(context.Background(), statement)+ require.NoError(t, err)+ require.Equal(t, int64(100), got)+ var statements []string+ for _, s := range testMySQLStatements {+ statements = append(statements, s.query)+ require.Equal(t, testMySQLStatements[0].conn, s.conn)+ }+ require.Equal(t, tc.wantStatements, statements)+ })}- defer func() { testMySQLExplainJSON = "" }()- db, err := sql.Open("test_mysql_explain", "")- require.NoError(t, err)- defer db.Close()+ t.Run("other SET failures stop the estimate", func(t *testing.T) {+ d := newTestMySQLDriver(t, testInsertSelectPlan)+ testMySQLSetErr = &mysql.MySQLError{Number: 1227, Message: "Access denied"}- driver := &Driver{db: db}+ _, err := d.CountAffectedRows(context.Background(), statement)+ require.ErrorContains(t, err, "failed to set explain_json_format_version")+ require.Len(t, testMySQLStatements, 1)+ })+}- got, err := driver.CountAffectedRows(context.Background(), "UPDATE o SET c = 1 WHERE c IS NULL AND EXISTS (SELECT 1 FROM t WHERE t.k = o.k);")- require.NoError(t, err)- require.Equal(t, int64(1144), got)+func TestCountAffectedRowsFallsBackToTabularPlan(t *testing.T) {+ // MySQL 8.0 prints no source plan for an INSERT ... SELECT that computes window functions.+ const plan = `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`+ for _, tc := range []struct {+ name string+ statement string+ // table is the tabular plan, which defaults to a single table with 1000 rows and 33.33% filtered.+ table [][]driver.Value+ want int64+ }{+ {+ name: "rows of the first table estimated, scaled by filtered",+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER (ORDER BY id) FROM t WHERE c < 10;",+ want: 333,+ },+ {+ name: "capped by LIMIT",+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER (ORDER BY id) FROM t WHERE c < 10 LIMIT 50;",+ want: 50,+ },+ {+ // 3 small rows each join 100 big rows; the subquery's own block does not count.+ name: "rows joined in the first query block",+ statement: "INSERT INTO t2 SELECT big.id, ROW_NUMBER() OVER () FROM small JOIN big ON big.s_id = small.id WHERE small.id IN (SELECT id FROM s);",+ table: [][]driver.Value{+ {int64(1), "INSERT", "t2", nil, nil},+ {int64(1), "SIMPLE", "small", int64(3), 100.0},+ {int64(1), "SIMPLE", "big", int64(100), 100.0},+ {int64(2), "SUBQUERY", "s", int64(100), 10.0},+ },+ want: 300,+ },+ {+ // Each of the 300 joined rows changes a row of a and a row of b.+ name: "multi-table update counts each target",+ statement: "UPDATE small JOIN big ON big.s_id = small.id SET small.x = 1, big.v = 1;",+ table: [][]driver.Value{+ {int64(1), "UPDATE", "small", int64(3), 100.0},+ {int64(1), "UPDATE", "big", int64(100), 100.0},+ },+ want: 600,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ d := newTestMySQLDriver(t, plan)+ testMySQLExplainTable = tc.table+ if testMySQLExplainTable == nil {+ testMySQLExplainTable = [][]driver.Value{+ {int64(1), "INSERT", "t2", nil, nil},+ {int64(1), "SIMPLE", "t", int64(1000), 33.33},+ }+ }+ got, err := d.CountAffectedRows(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ require.Equal(t, "EXPLAIN FORMAT=TRADITIONAL "+tc.statement, testMySQLStatements[len(testMySQLStatements)-1].query)+ })+ }+}- // The statement LIMIT still caps the JSON-plan estimate.- got, err = driver.CountAffectedRows(context.Background(), "UPDATE o SET c = 1 WHERE c IS NULL LIMIT 10;")- require.NoError(t, err)- require.Equal(t, int64(10), got)+func TestCountAffectedRowsReportsUninterpretablePlan(t *testing.T) {+ // A plan without an estimate, with no tabular estimate either, fails rather than reading as zero rows.+ d := newTestMySQLDriver(t, `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`)+ _, err := d.CountAffectedRows(context.Background(), "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;")+ require.ErrorContains(t, err, `table "t2" in the plan has no row estimate`)}-func TestCountAffectedRowsForOceanBaseConcatenatesExplainRows(t *testing.T) {+func TestCountAffectedRowsForOceanBase(t *testing.T) {+ db, err := sql.Open("test_oceanbase_explain", "")+ require.NoError(t, err)+ defer db.Close()+ d := &Driver{dbType: storepb.Engine_OCEANBASE, db: db}++ // OceanBase splits the JSON plan across rows.testOceanBaseExplainRows = [][]driver.Value{{`{`},{` "ID":0,`},@@ -212,11 +371,15 @@ func TestCountAffectedRowsForOceanBaseConcatenatesExplainRows(t *testing.T) {{` }`},{`}`},}- db, err := sql.Open("test_oceanbase_explain", "")- require.NoError(t, err)- defer db.Close()-- got, err := countAffectedRowsForOceanBase(context.Background(), db, "update dba_test_1 set log_id=1 where id < 1000;")+ got, err := d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")require.NoError(t, err)require.Equal(t, int64(1000), got)++ testOceanBaseExplainRows = [][]driver.Value{{`{"ID":0,"OPERATOR":"UPDATE","NAME":""}`}}+ _, err = d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")+ require.ErrorContains(t, err, `operator "UPDATE" has no EST.ROWS`)++ testOceanBaseExplainRows = nil+ _, err = d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")+ require.Error(t, err)}diff --git a/backend/plugin/db/oracle/stats.go b/backend/plugin/db/oracle/stats.goindex d7231b8582..3c1dcaed6d 100644--- a/backend/plugin/db/oracle/stats.go+++ b/backend/plugin/db/oracle/stats.go@@ -2,67 +2,196 @@ package oracleimport ("context"+ "crypto/rand""database/sql""fmt"- "strconv"+ "log/slog"+ "math/big"+ "slices""strings"+ "time""github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/utils")+// dynamicSamplingHint makes the optimizer sample the tables while it plans the statement, so its+// estimates account for skewed columns without histograms and for correlated predicates. Unlike a+// DYNAMIC_SAMPLING hint, which covers only its own query block, OPT_PARAM covers every block.+const dynamicSamplingHint = "OPT_PARAM('optimizer_dynamic_sampling' 11)"++// CountAffectedRows returns the optimizer's estimate of the rows an UPDATE, DELETE, INSERT, or MERGE+// changes, planned with dynamic sampling.func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {- // Statement includes trailing semicolon, so we need to remove it.- if _, err := d.db.ExecContext(ctx, fmt.Sprintf("EXPLAIN PLAN FOR %s", statement)); err != nil {- return 0, err+ // Older Oracle releases reject a trailing semicolon with ORA-00911.+ statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon)+ randNum, err := rand.Int(rand.Reader, big.NewInt(999))+ if err != nil {+ return 0, errors.Wrapf(err, "failed to generate random statement ID")}- rows, err := d.db.QueryContext(ctx, "SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY)")+ statementID := fmt.Sprintf("%d%d", time.Now().UnixMilli(), randNum.Int64())++ // The default PLAN_TABLE is a session-private temporary table, so explaining,+ // reading, and deleting the plan must share one connection.+ conn, err := d.db.Conn(ctx)if err != nil {- return 0, err+ return 0, errors.Wrapf(err, "failed to get connection")}- defer rows.Close()+ defer conn.Close()- // EXPLAIN PLAN FOR UPDATE TEST SET id = 121 WHERE id = 242; SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY);- // | Id | Operation | Name | Rows | Bytes | Cost (%CPU)| Time |- // ---------------------------------------------------------------------------- // | 0 | UPDATE STATEMENT | | 1 | 13 | 3 (0)| 00:00:01 |- // | 1 | UPDATE | TEST | | | | |- // |* 2 | TABLE ACCESS FULL| TEST | 1 | 13 | 3 (0)| 00:00:01 |- var rowsIndex int- foundRowsIndex := false+ if _, err := conn.ExecContext(ctx, fmt.Sprintf("EXPLAIN PLAN SET STATEMENT_ID = '%s' FOR %s", statementID, withDynamicSamplingHint(statement))); err != nil {+ return 0, errors.Wrapf(err, "failed to explain statement")+ }+ defer func() {+ if _, err := conn.ExecContext(ctx, fmt.Sprintf("DELETE FROM PLAN_TABLE WHERE STATEMENT_ID = '%s'", statementID)); err != nil {+ slog.Warn("failed to delete explain plan rows", log.BBError(err))+ }+ }()++ rows, err := conn.QueryContext(ctx, fmt.Sprintf("SELECT ID, PARENT_ID, OPERATION, CARDINALITY FROM PLAN_TABLE WHERE STATEMENT_ID = '%s' ORDER BY ID", statementID))+ if err != nil {+ return 0, errors.Wrapf(err, "failed to get plan cardinality")+ }+ defer rows.Close()+ var plan []planRowfor rows.Next() {- var planRow sql.NullString- if err := rows.Scan(&planRow); err != nil {- return 0, err+ var row planRow+ if err := rows.Scan(&row.id, &row.parentID, &row.operation, &row.cardinality); err != nil {+ return 0, errors.Wrapf(err, "failed to get plan cardinality")}- if !planRow.Valid {+ plan = append(plan, row)+ }+ if err := rows.Err(); err != nil {+ return 0, errors.Wrapf(err, "failed to get plan cardinality")+ }+ return getAffectedRowsFromPlan(plan, isInsertFirst(statement))+}++type planRow struct {+ id int64+ parentID sql.NullInt64+ operation string+ cardinality sql.NullInt64+}++// getAffectedRowsFromPlan returns the cardinality of the statement row (ID 0). For a MERGE it returns+// the cardinality of the rows the MERGE operation reads instead, because with dynamic sampling the+// statement row of a MERGE plan no longer estimates them. A multi-table INSERT ALL can write each row+// it reads to every INTO operation, so its estimate is the statement row times those operations;+// insertFirst marks an INSERT FIRST, which writes each row at most once.+func getAffectedRowsFromPlan(plan []planRow, insertFirst bool) (int64, error) {+ for _, row := range plan {+ if row.operation != "MERGE" {continue}- tokens := strings.Split(planRow.String, "|")- if !foundRowsIndex {- for i, token := range tokens {- token = strings.Trim(token, " ")- if token == "Rows" {- rowsIndex, foundRowsIndex = i, true- break- }+ // The first child chain below MERGE passes through views that have no cardinality.+ for parentID := row.id; ; {+ i := slices.IndexFunc(plan, func(r planRow) bool { return r.parentID.Valid && r.parentID.Int64 == parentID })+ if i < 0 || plan[i].id <= parentID {+ return 0, errors.New("the MERGE operation in the plan has no cardinality estimate")}- continue- }- if rowsIndex >= len(tokens) {- continue- }- rowsToken := strings.Trim(tokens[rowsIndex], " ")- if rowsToken == "" {- continue+ if plan[i].cardinality.Valid {+ return plan[i].cardinality.Int64, nil+ }+ parentID = plan[i].id}- v, err := strconv.ParseInt(rowsToken, 10, 64)- if err != nil {- return 0, errors.Errorf("failed to get integer from %q", rowsToken)+ }+ i := slices.IndexFunc(plan, func(r planRow) bool { return r.id == 0 })+ if i < 0 {+ return 0, errors.New("plan has no statement row")+ }+ if !plan[i].cardinality.Valid {+ return 0, errors.New("plan has no cardinality estimate")+ }+ rows := plan[i].cardinality.Int64+ targets := 0+ for _, row := range plan {+ if row.operation == "INTO" {+ targets++}- return v, nil}- if err := rows.Err(); err != nil {- return 0, err+ if targets > 1 && !insertFirst {+ return common.RoundRows(float64(rows) * float64(targets)), nil+ }+ return rows, nil+}++// isInsertFirst reports whether statement is an INSERT FIRST.+func isInsertFirst(statement string) bool {+ start := skipSpaceAndComments(statement)+ end := start+ for end < len(statement) && isLetter(statement[end]) {+ end+++ }+ if !strings.EqualFold(statement[start:end], "INSERT") {+ return false+ }+ start = end + skipSpaceAndComments(statement[end:])+ end = start+ for end < len(statement) && isLetter(statement[end]) {+ end+++ }+ return strings.EqualFold(statement[start:end], "FIRST")+}++// withDynamicSamplingHint adds dynamicSamplingHint to the hint of an UPDATE, DELETE, INSERT, or MERGE+// statement and returns other statements unchanged.+func withDynamicSamplingHint(statement string) string {+ start := skipSpaceAndComments(statement)+ end := start+ for end < len(statement) && isLetter(statement[end]) {+ end+++ }+ switch strings.ToUpper(statement[start:end]) {+ case "UPDATE", "DELETE", "INSERT", "MERGE":+ default:+ return statement}- return 0, nil+ // Oracle reads only the first hint after the keyword, so the hint joins an existing one.+ hint := end+ for hint < len(statement) && isSpace(statement[hint]) {+ hint+++ }+ if strings.HasPrefix(statement[hint:], "/*+") || strings.HasPrefix(statement[hint:], "--+") {+ return statement[:hint+3] + " " + dynamicSamplingHint + " " + statement[hint+3:]+ }+ return statement[:end] + " /*+ " + dynamicSamplingHint + " */" + statement[end:]+}++// skipSpaceAndComments returns the offset of the first byte of the statement that is not whitespace+// or part of a comment.+func skipSpaceAndComments(statement string) int {+ i := 0+ for i < len(statement) {+ switch {+ case isSpace(statement[i]):+ i+++ case strings.HasPrefix(statement[i:], "--"):+ end := strings.IndexByte(statement[i:], '\n')+ if end < 0 {+ return len(statement)+ }+ i += end + 1+ case strings.HasPrefix(statement[i:], "/*"):+ end := strings.Index(statement[i+2:], "*/")+ if end < 0 {+ return len(statement)+ }+ i += end + 4+ default:+ return i+ }+ }+ return i+}++func isSpace(c byte) bool {+ return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'+}++func isLetter(c byte) bool {+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')}diff --git a/backend/plugin/db/oracle/stats_test.go b/backend/plugin/db/oracle/stats_test.gonew file mode 100644index 0000000000..70b066350d--- /dev/null+++ b/backend/plugin/db/oracle/stats_test.go@@ -0,0 +1,247 @@+package oracle++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "io"+ "regexp"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestCountAffectedRows(t *testing.T) {+ const hint = "/*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */"+ for _, tc := range []struct {+ name string+ statement string+ // plan holds the ID, PARENT_ID, OPERATION, and CARDINALITY of each PLAN_TABLE row.+ plan [][]driver.Value+ explained string+ want int64+ wantErr bool+ }{+ {+ name: "large_cardinality",+ statement: "UPDATE huge SET c = 1 WHERE c < 10",+ // go-ora returns NUMBER columns as strings.+ plan: [][]driver.Value{+ {"0", nil, "UPDATE STATEMENT", "5000000"},+ {"1", "0", "UPDATE", nil},+ {"2", "1", "TABLE ACCESS", "5000000"},+ },+ explained: "UPDATE " + hint + " huge SET c = 1 WHERE c < 10",+ want: 5000000,+ },+ {+ name: "trailing_semicolon",+ statement: "DELETE FROM t WHERE c < 10;\n",+ plan: [][]driver.Value{{"0", nil, "DELETE STATEMENT", "101"}},+ explained: "DELETE " + hint + " FROM t WHERE c < 10",+ want: 101,+ },+ {+ // Captured from Oracle 23 with dynamic sampling, where the statement row estimates 1 row+ // for a MERGE that inserts 100.+ name: "merge_reads_the_rows_below_the_merge_operation",+ statement: "MERGE INTO t2 USING (SELECT * FROM t WHERE c < 10) src ON (t2.id = src.id) WHEN NOT MATCHED THEN INSERT VALUES (src.id, src.c, src.d, src.grp, src.v)",+ plan: [][]driver.Value{+ {"0", nil, "MERGE STATEMENT", "1"},+ {"1", "0", "MERGE", nil},+ {"2", "1", "VIEW", nil},+ {"3", "2", "NESTED LOOPS", "100"},+ {"4", "3", "TABLE ACCESS", "100"},+ {"5", "3", "TABLE ACCESS", "1"},+ {"6", "5", "INDEX", "1"},+ },+ explained: "MERGE " + hint + " INTO t2 USING (SELECT * FROM t WHERE c < 10) src ON (t2.id = src.id) WHEN NOT MATCHED THEN INSERT VALUES (src.id, src.c, src.d, src.grp, src.v)",+ want: 100,+ },+ {+ // Captured from Oracle 23, where the statement row estimates the 1000 rows an INSERT ALL reads+ // and writes to three tables.+ name: "insert_all_writes_each_row_to_every_into",+ statement: "INSERT ALL INTO t1 VALUES (id) INTO t2 VALUES (id) INTO t3 VALUES (id) SELECT id FROM src",+ plan: [][]driver.Value{+ {"0", nil, "INSERT STATEMENT", "1000"},+ {"1", "0", "MULTI-TABLE INSERT", nil},+ {"2", "1", "TABLE ACCESS", "1000"},+ {"3", "1", "INTO", nil},+ {"4", "1", "INTO", nil},+ {"5", "1", "INTO", nil},+ },+ explained: "INSERT " + hint + " ALL INTO t1 VALUES (id) INTO t2 VALUES (id) INTO t3 VALUES (id) SELECT id FROM src",+ want: 3000,+ },+ {+ name: "insert_first_writes_each_row_once",+ statement: "INSERT /* move */ FIRST WHEN id < 10 THEN INTO t1 VALUES (id) ELSE INTO t2 VALUES (id) SELECT id FROM src",+ plan: [][]driver.Value{+ {"0", nil, "INSERT STATEMENT", "1000"},+ {"1", "0", "MULTI-TABLE INSERT", nil},+ {"2", "1", "TABLE ACCESS", "1000"},+ {"3", "1", "INTO", nil},+ {"4", "1", "INTO", nil},+ },+ explained: "INSERT " + hint + " /* move */ FIRST WHEN id < 10 THEN INTO t1 VALUES (id) ELSE INTO t2 VALUES (id) SELECT id FROM src",+ want: 1000,+ },+ {+ name: "merge_without_cardinality",+ statement: "MERGE INTO t2 USING t ON (t2.id = t.id) WHEN MATCHED THEN UPDATE SET t2.v = t.v",+ plan: [][]driver.Value{+ {"0", nil, "MERGE STATEMENT", "1000"},+ {"1", "0", "MERGE", nil},+ {"2", "1", "VIEW", nil},+ },+ explained: "MERGE " + hint + " INTO t2 USING t ON (t2.id = t.id) WHEN MATCHED THEN UPDATE SET t2.v = t.v",+ wantErr: true,+ },+ {+ name: "null_cardinality",+ statement: "DELETE FROM t;",+ plan: [][]driver.Value{{"0", nil, "DELETE STATEMENT", nil}},+ explained: "DELETE " + hint + " FROM t",+ wantErr: true,+ },+ {+ name: "missing_statement_row",+ statement: "DELETE FROM t;",+ explained: "DELETE " + hint + " FROM t",+ wantErr: true,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ connector := &fakePlanConnector{plan: tc.plan}+ db := sql.OpenDB(connector)+ defer db.Close()+ // Without idle connections, every statement that is not pinned to one connection opens a new one.+ db.SetMaxIdleConns(0)++ got, err := (&Driver{db: db}).CountAffectedRows(context.Background(), tc.statement)+ if tc.wantErr {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ }++ require.NotEmpty(t, connector.queries)+ match := regexp.MustCompile(`^EXPLAIN PLAN SET STATEMENT_ID = '(\d+)' FOR `).FindStringSubmatch(connector.queries[0].text)+ require.NotNil(t, match, connector.queries[0].text)+ id := match[1]+ require.Equal(t, []fakePlanQuery{+ {conn: 1, text: "EXPLAIN PLAN SET STATEMENT_ID = '" + id + "' FOR " + tc.explained},+ {conn: 1, text: "SELECT ID, PARENT_ID, OPERATION, CARDINALITY FROM PLAN_TABLE WHERE STATEMENT_ID = '" + id + "' ORDER BY ID"},+ {conn: 1, text: "DELETE FROM PLAN_TABLE WHERE STATEMENT_ID = '" + id + "'"},+ }, connector.queries)+ })+ }+}++func TestWithDynamicSamplingHint(t *testing.T) {+ for _, tc := range []struct {+ statement string+ want string+ }{+ {+ statement: "update t set c = 1",+ want: "update /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ t set c = 1",+ },+ {+ statement: "INSERT INTO t2 SELECT * FROM t",+ want: "INSERT /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ INTO t2 SELECT * FROM t",+ },+ {+ statement: "-- clean up\n/* ticket 42 */ DELETE FROM t",+ want: "-- clean up\n/* ticket 42 */ DELETE /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ FROM t",+ },+ {+ statement: "UPDATE /*+ INDEX(t t_c) */ t SET v = 1 WHERE c = 1",+ want: "UPDATE /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) INDEX(t t_c) */ t SET v = 1 WHERE c = 1",+ },+ {+ statement: "DELETE --+ FULL(t)\nFROM t",+ want: "DELETE --+ OPT_PARAM('optimizer_dynamic_sampling' 11) FULL(t)\nFROM t",+ },+ {+ statement: "SELECT * FROM t",+ want: "SELECT * FROM t",+ },+ } {+ t.Run(tc.statement, func(t *testing.T) {+ require.Equal(t, tc.want, withDynamicSamplingHint(tc.statement))+ })+ }+}++// fakePlanConnector returns the plan rows for every query and records each statement with the+// number of the connection that ran it.+type fakePlanConnector struct {+ plan [][]driver.Value+ opened int+ queries []fakePlanQuery+}++type fakePlanQuery struct {+ conn int+ text string+}++func (c *fakePlanConnector) Connect(context.Context) (driver.Conn, error) {+ c.opened+++ return &fakePlanConn{connector: c, id: c.opened}, nil+}++func (*fakePlanConnector) Driver() driver.Driver {+ return nil+}++type fakePlanConn struct {+ connector *fakePlanConnector+ id int+}++func (*fakePlanConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (*fakePlanConn) Close() error {+ return nil+}++func (*fakePlanConn) Begin() (driver.Tx, error) {+ return nil, driver.ErrSkip+}++func (c *fakePlanConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {+ c.connector.queries = append(c.connector.queries, fakePlanQuery{conn: c.id, text: query})+ return driver.RowsAffected(0), nil+}++func (c *fakePlanConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {+ c.connector.queries = append(c.connector.queries, fakePlanQuery{conn: c.id, text: query})+ return &fakePlanRows{rows: c.connector.plan}, nil+}++type fakePlanRows struct {+ rows [][]driver.Value+}++func (*fakePlanRows) Columns() []string {+ return []string{"ID", "PARENT_ID", "OPERATION", "CARDINALITY"}+}++func (*fakePlanRows) Close() error {+ return nil+}++func (r *fakePlanRows) Next(dest []driver.Value) error {+ if len(r.rows) == 0 {+ return io.EOF+ }+ copy(dest, r.rows[0])+ r.rows = r.rows[1:]+ return nil+}diff --git a/backend/plugin/db/pg/stats.go b/backend/plugin/db/pg/stats.goindex cdc1ca4da6..9606ab59c1 100644--- a/backend/plugin/db/pg/stats.go+++ b/backend/plugin/db/pg/stats.go@@ -2,59 +2,31 @@ package pgimport ("context"- "database/sql"- "fmt"- "regexp"- "strconv"- "github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")-var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")-+// CountAffectedRows returns the planner's estimate of the rows the statement modifies. A search+// path that base.WithSearchPath put before the statement is set for the EXPLAIN in a transaction+// that is rolled back.func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)- rows, err := d.db.QueryContext(ctx, explainSQL)- if err != nil {- return 0, err- }- defer rows.Close()-- // test-bb=# EXPLAIN INSERT INTO t SELECT * FROM t;- // QUERY PLAN- // -------------------------------------------------------------- // Insert on t (cost=0.00..1.03 rows=0 width=0)- // -> Seq Scan on t t_1 (cost=0.00..1.03 rows=3 width=520)- // (2 rows)- //- // d1=# explain select * from h1;- // QUERY PLAN- // ------------------------------------------------------- // Seq Scan on h1 (cost=0.00..35.50 rows=2550 width=4)- // (1 row)- var rowCount int64- for rows.Next() {- var planRow sql.NullString- if err := rows.Scan(&planRow); err != nil {+ setup, statement := base.SplitSearchPath(statement)+ queryRow := d.db.QueryRowContext+ if setup != "" {+ tx, err := d.db.BeginTx(ctx, nil)+ if err != nil {return 0, err}-- if !planRow.Valid {- continue- }-- matches := rowsRegexp.FindStringSubmatch(planRow.String)- if len(matches) != 2 {- continue- }- v, err := strconv.ParseInt(matches[1], 10, 64)- if err != nil {- return 0, errors.Errorf("failed to get integer from %q", matches[1])+ defer tx.Rollback()+ if _, err := tx.ExecContext(ctx, setup); err != nil {+ return 0, err}- rowCount = v+ queryRow = tx.QueryRowContext}- if err := rows.Err(); err != nil {+ var plan string+ if err := queryRow(ctx, "EXPLAIN (FORMAT JSON) "+statement).Scan(&plan); err != nil {return 0, err}- return rowCount, nil+ return pgparser.GetEstimatedAffectedRowsFromExplainJSON(plan)}diff --git a/backend/plugin/db/pg/stats_test.go b/backend/plugin/db/pg/stats_test.gonew file mode 100644index 0000000000..8a356666f5--- /dev/null+++ b/backend/plugin/db/pg/stats_test.go@@ -0,0 +1,76 @@+package pg++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestCountAffectedRows(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ pgContainer := testcontainer.SharedPgContainer(t)+ dbName, rawDB := testcontainer.NewPgDatabase(t)+ _, err := rawDB.ExecContext(ctx, `+ CREATE TABLE s (id int PRIMARY KEY, flag bool NOT NULL);+ INSERT INTO s SELECT g, g <= 10 FROM generate_series(1, 100) g;+ CREATE TABLE big (id int PRIMARY KEY, s_id int NOT NULL, v int);+ INSERT INTO big SELECT g, (g % 100) + 1, 0 FROM generate_series(1, 10000) g;+ CREATE TABLE big_archive (LIKE big);+ ANALYZE;+ `)+ require.NoError(t, err)++ openedDriver, err := (&Driver{}).Open(ctx, storepb.Engine_POSTGRES, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Host: pgContainer.GetHost(),+ Port: pgContainer.GetPort(),+ Username: "postgres",+ },+ Password: "root-password",+ ConnectionContext: db.ConnectionContext{DatabaseName: dbName},+ })+ require.NoError(t, err)+ defer openedDriver.Close(ctx)+ driver, ok := openedDriver.(*Driver)+ require.True(t, ok)++ // Each statement modifies the 1000 big rows joined to the 10 flagged s rows.+ for _, statement := range []string{+ "DELETE FROM big USING s WHERE big.s_id = s.id AND s.flag;",+ "UPDATE big SET v = 1 FROM s WHERE big.s_id = s.id AND s.flag;",+ "DELETE FROM big WHERE s_id IN (SELECT id FROM s WHERE flag);",+ "INSERT INTO big_archive SELECT big.* FROM big JOIN s ON big.s_id = s.id WHERE s.flag;",+ "MERGE INTO big USING s ON big.s_id = s.id AND s.flag WHEN MATCHED THEN UPDATE SET v = 1;",+ } {+ rows, err := driver.CountAffectedRows(ctx, statement)+ require.NoError(t, err, statement)+ require.Greater(t, rows, int64(500), statement)+ require.LessOrEqual(t, rows, int64(2000), statement)+ }++ _, err = driver.CountAffectedRows(ctx, "SELECT * FROM big;")+ require.Error(t, err)++ _, err = rawDB.ExecContext(ctx, `+ CREATE SCHEMA app;+ CREATE TABLE app.only_in_app (id int);+ INSERT INTO app.only_in_app SELECT generate_series(1, 100);+ ANALYZE app.only_in_app;+ `)+ require.NoError(t, err)+ // With one connection, the second estimate shows that the search path ended with the first.+ driver.db.SetMaxOpenConns(1)+ rows, err := driver.CountAffectedRows(ctx, base.WithSearchPath("DELETE FROM only_in_app;", []string{"app"}))+ require.NoError(t, err)+ require.Equal(t, int64(100), rows)+ _, err = driver.CountAffectedRows(ctx, "DELETE FROM only_in_app;")+ require.ErrorContains(t, err, "does not exist")+}diff --git a/backend/plugin/db/redshift/stats.go b/backend/plugin/db/redshift/stats.goindex 8fbd4e7134..920fbb25ff 100644--- a/backend/plugin/db/redshift/stats.go+++ b/backend/plugin/db/redshift/stats.go@@ -3,55 +3,69 @@ package redshiftimport ("context""database/sql"- "fmt""regexp""strconv""github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/plugin/parser/base")var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")+// CountAffectedRows returns the planner's estimate of the rows the statement modifies. A search+// path that base.WithSearchPath put before the statement is set for the EXPLAIN in a transaction+// that is rolled back.func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)- rows, err := d.db.QueryContext(ctx, explainSQL)+ setup, statement := base.SplitSearchPath(statement)+ query := d.db.QueryContext+ if setup != "" {+ tx, err := d.db.BeginTx(ctx, nil)+ if err != nil {+ return 0, err+ }+ defer tx.Rollback()+ if _, err := tx.ExecContext(ctx, setup); err != nil {+ return 0, err+ }+ query = tx.QueryContext+ }+ rows, err := query(ctx, "EXPLAIN "+statement)if err != nil {return 0, err}defer rows.Close()- // # EXPLAIN INSERT INTO t SELECT * FROM t;- // QUERY PLAN- // -------------------------------------------------------------- // XN Seq Scan on t (cost=0.00..0.11 rows=11 width=49)- //- // # EXPLAIN SELECT * FROM t;- // QUERY PLAN- // ------------------------------------------------------- // XN Seq Scan on t (cost=0.00..0.11 rows=11 width=49)- var rowCount int64+ var plan []stringfor rows.Next() {- var planRow sql.NullString- if err := rows.Scan(&planRow); err != nil {+ var line sql.NullString+ if err := rows.Scan(&line); err != nil {return 0, err}-- if !planRow.Valid {- continue+ if line.Valid {+ plan = append(plan, line.String)}+ }+ if err := rows.Err(); err != nil {+ return 0, err+ }+ return getAffectedRowsFromPlan(plan)+}- matches := rowsRegexp.FindStringSubmatch(planRow.String)+// getAffectedRowsFromPlan returns the rows estimate of the top plan node. Redshift+// plans have no DML node, so an INSERT, UPDATE, or DELETE shows only the scans and+// joins that produce its rows, and the nodes below the top estimate their inputs.+func getAffectedRowsFromPlan(plan []string) (int64, error) {+ for _, line := range plan {+ matches := rowsRegexp.FindStringSubmatch(line)if len(matches) != 2 {continue}- v, err := strconv.ParseInt(matches[1], 10, 64)+ rows, err := strconv.ParseInt(matches[1], 10, 64)if err != nil {- return 0, errors.Errorf("failed to get integer from %q", matches[1])+ return 0, errors.Wrapf(err, "failed to parse rows estimate in %q", line)}- rowCount = v- }- if err := rows.Err(); err != nil {- return 0, err+ return rows, nil}- return rowCount, nil+ return 0, errors.Errorf("EXPLAIN plan has no rows estimate: %q", plan)}diff --git a/backend/plugin/db/redshift/stats_test.go b/backend/plugin/db/redshift/stats_test.gonew file mode 100644index 0000000000..086cc7787b--- /dev/null+++ b/backend/plugin/db/redshift/stats_test.go@@ -0,0 +1,77 @@+package redshift++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+)++// The plans are EXPLAIN output published in the Amazon Redshift docs:+// https://docs.aws.amazon.com/redshift/latest/dg/c-the-query-plan.html and+// https://docs.aws.amazon.com/redshift/latest/dg/r_EXPLAIN.html.+func TestGetAffectedRowsFromPlan(t *testing.T) {+ for _, tc := range []struct {+ name string+ plan string+ want int64+ wantErr bool+ }{+ {+ name: "hash_join_estimates_joined_rows_not_its_inner_table",+ plan: ` XN Hash Join DS_BCAST_INNER (cost=0.14..6600286.07 rows=8798 width=84)+ Hash Cond: ("outer".catid = "inner".catid)+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=35)+ -> XN Hash (cost=0.11..0.11 rows=11 width=49)+ -> XN Seq Scan on category (cost=0.00..0.11 rows=11 width=49)`,+ want: 8798,+ },+ {+ name: "three_way_join_estimates_the_outermost_join",+ plan: `XN Hash Join DS_BCAST_INNER (cost=109.98..3871130276.17 rows=172456 width=132)+ Hash Cond: ("outer".eventid = "inner".eventid)+ -> XN Merge Join DS_DIST_NONE (cost=0.00..6285.93 rows=172456 width=97)+ Merge Cond: ("outer".listid = "inner".listid)+ -> XN Seq Scan on listing (cost=0.00..1924.97 rows=192497 width=44)+ -> XN Seq Scan on sales (cost=0.00..1724.56 rows=172456 width=53)+ -> XN Hash (cost=87.98..87.98 rows=8798 width=35)+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=35)`,+ want: 172456,+ },+ {+ name: "aggregate_estimates_groups_not_scanned_rows",+ plan: `XN HashAggregate (cost=131.97..133.41 rows=576 width=17)+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=17)`,+ want: 576,+ },+ {+ name: "scan_estimates_rows_after_its_filter",+ plan: `XN Seq Scan on venue (cost=0.00..2.02 rows=187 width=45)+Filter: (venueseats IS NOT NULL)`,+ want: 187,+ },+ {+ name: "plan_without_rows_estimate",+ plan: `Filter: (venueseats IS NOT NULL)`,+ wantErr: true,+ },+ {+ name: "empty_plan",+ wantErr: true,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ var plan []string+ if tc.plan != "" {+ plan = strings.Split(tc.plan, "\n")+ }+ got, err := getAffectedRowsFromPlan(plan)+ if tc.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ })+ }+}diff --git a/backend/plugin/db/tidb/stats.go b/backend/plugin/db/tidb/stats.goindex 0572ee9250..6150732fb4 100644--- a/backend/plugin/db/tidb/stats.go+++ b/backend/plugin/db/tidb/stats.go@@ -2,60 +2,144 @@ package tidbimport ("context"- "database/sql""fmt"+ "slices"+ "strconv"+ "strings"+ tidbast "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/plugin/db/util"+ tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb")func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)- rows, err := d.db.QueryContext(ctx, explainSQL)+ rows, err := d.db.QueryContext(ctx, fmt.Sprintf("EXPLAIN %s", statement))if err != nil {return 0, err}defer rows.Close()- // mysql> explain delete from td;- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- // | 1 | DELETE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | NULL |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+- //- // mysql> explain insert into td select * from td;- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+- // | 1 | INSERT | td | NULL | ALL | NULL | NULL | NULL | NULL | NULL | NULL | NULL |- // | 1 | SIMPLE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | Using temporary |- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+columns, err := rows.Columns()if err != nil {return 0, err}- rowsIndex, ok := util.GetColumnIndex(columns, "rows")+ idIndex, ok := util.GetColumnIndex(columns, "id")if !ok {- return 0, nil+ return 0, errors.Errorf("EXPLAIN returned no id column: %v", columns)+ }+ estRowsIndex, ok := util.GetColumnIndex(columns, "estRows")+ if !ok {+ return 0, errors.Errorf("EXPLAIN returned no estRows column: %v", columns)}+ var plan []planRowfor rows.Next() {scanArgs := make([]any, len(columns))for i := range scanArgs {var unused anyscanArgs[i] = &unused}- var rowsColumn sql.NullInt64- scanArgs[rowsIndex] = &rowsColumn+ var row planRow+ scanArgs[idIndex] = &row.id+ scanArgs[estRowsIndex] = &row.estRowsif err := rows.Scan(scanArgs...); err != nil {return 0, err}-- if rowsColumn.Valid {- return rowsColumn.Int64, nil- }+ plan = append(plan, row)}if err := rows.Err(); err != nil {return 0, err}- return 0, nil+ return getAffectedRows(plan, statement)+}++// getAffectedRows returns the rows statement modifies according to its plan. The plan estimates the+// rows the statement reads, and a multi-table UPDATE or DELETE can change a row of each target table+// for every one of them.+func getAffectedRows(plan []planRow, statement string) (int64, error) {+ estimate, err := getAffectedRowsFromPlan(plan)+ if err != nil {+ return 0, err+ }+ return common.RoundRows(estimate * float64(countDMLTargets(statement))), nil+}++// countDMLTargets returns how many tables an UPDATE or DELETE can change for each row it reads: the+// targets a multi-table DELETE lists, or the tables its assignments qualify, counting an unqualified+// assignment as another joined table, up to the number of joined tables. Other statements change one.+func countDMLTargets(statement string) int {+ nodes, err := tidbparser.ParseTiDB(statement, "", "")+ if err != nil || len(nodes) != 1 {+ return 1+ }+ switch n := nodes[0].(type) {+ case *tidbast.DeleteStmt:+ if n.IsMultiTable && n.Tables != nil {+ return max(len(n.Tables.Tables), 1)+ }+ case *tidbast.UpdateStmt:+ if n.TableRefs == nil {+ return 1+ }+ var tables []string+ unqualified := 0+ for _, assignment := range n.List {+ switch {+ case assignment.Column == nil || assignment.Column.Table.L == "":+ unqualified+++ case !slices.Contains(tables, assignment.Column.Table.L):+ tables = append(tables, assignment.Column.Table.L)+ default:+ }+ }+ return max(min(len(tables)+unqualified, countTableSources(n.TableRefs.TableRefs)), 1)+ default:+ }+ return 1+}++func countTableSources(node tidbast.ResultSetNode) int {+ switch n := node.(type) {+ case *tidbast.Join:+ count := countTableSources(n.Left)+ if n.Right != nil {+ count += countTableSources(n.Right)+ }+ return count+ case *tidbast.TableSource:+ return 1+ default:+ return 0+ }+}++type planRow struct {+ id string+ estRows string+}++// getAffectedRowsFromPlan returns the estRows of the operator that feeds rows to the+// INSERT, UPDATE, or DELETE at the plan root. The root itself reports N/A, operators+// deeper in the tree estimate rows before the filters, joins, and limits above them+// apply, and a later root plans a CTE or subquery.+func getAffectedRowsFromPlan(plan []planRow) (float64, error) {+ if len(plan) == 0 {+ return 0, errors.New("EXPLAIN returned no plan")+ }+ if operator, _, _ := strings.Cut(plan[0].id, "_"); operator != "Insert" && operator != "Update" && operator != "Delete" {+ return 0, errors.Errorf("EXPLAIN plan root %q is not an INSERT, UPDATE, or DELETE", plan[0].id)+ }+ // EXPLAIN lists operators in pre-order and draws each child of the root with a+ // leading "├─" or "└─", so the next row is the root's first child unless the root+ // has none, as for INSERT ... SET.+ if len(plan) == 1 || (!strings.HasPrefix(plan[1].id, "├─") && !strings.HasPrefix(plan[1].id, "└─")) {+ return 0, nil+ }+ estRows, err := strconv.ParseFloat(plan[1].estRows, 64)+ if err != nil {+ return 0, errors.Wrapf(err, "failed to parse estRows of %q", plan[1].id)+ }+ return estRows, nil}diff --git a/backend/plugin/db/tidb/stats_test.go b/backend/plugin/db/tidb/stats_test.gonew file mode 100644index 0000000000..8a55e18cc6--- /dev/null+++ b/backend/plugin/db/tidb/stats_test.go@@ -0,0 +1,177 @@+package tidb++import (+ "context"+ "fmt"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/testcontainer"+)++func TestCountAffectedRows(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ container, database := testcontainer.NewTiDBDatabase(t)++ driver := openTestDriver(ctx, t, container)+ t.Cleanup(func() {+ require.NoError(t, driver.Close(ctx))+ })++ // By default ANALYZE collects only columns that earlier queries filtered on,+ // which would leave c without statistics.+ _, err := container.GetDB().ExecContext(ctx, fmt.Sprintf(`+ CREATE TABLE %[1]s.t (id INT PRIMARY KEY, c INT);+ CREATE TABLE %[1]s.archive (id INT PRIMARY KEY, c INT);+ INSERT INTO %[1]s.t+ WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 1000)+ SELECT n, n %% 10 FROM seq;+ ANALYZE TABLE %[1]s.t ALL COLUMNS;+ `, database))+ require.NoError(t, err)++ for _, tc := range []struct {+ name string+ statement string+ }{+ {name: "update", statement: fmt.Sprintf("UPDATE %s.t SET c = 0 WHERE c = 1", database)},+ {name: "delete", statement: fmt.Sprintf("DELETE FROM %s.t WHERE c = 1", database)},+ {name: "insert_select", statement: fmt.Sprintf("INSERT INTO %[1]s.archive SELECT * FROM %[1]s.t WHERE c = 1", database)},+ } {+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ got, err := driver.CountAffectedRows(ctx, tc.statement)+ require.NoError(t, err)+ // 100 of the 1000 rows match c = 1, while the table scan beneath the+ // filter estimates all 1000.+ require.Positive(t, got)+ require.Less(t, got, int64(1000))+ })+ }+}++// The plans are EXPLAIN output captured from TiDB v8.5.0.+func TestGetAffectedRowsFromPlan(t *testing.T) {+ for _, tc := range []struct {+ name string+ plan []planRow+ want int64+ wantErr bool+ }{+ {+ name: "join_estimates_joined_rows_not_its_inputs",+ plan: []planRow{+ {"Update_8", "N/A"},+ {"└─HashJoin_27", "800.00"},+ {" ├─TableReader_44(Build)", "0.40"},+ {" │ └─Selection_43", "0.40"},+ {" │ └─TableFullScan_42", "400.00"},+ {" └─TableReader_38(Probe)", "1000.00"},+ {" └─Selection_37", "1000.00"},+ {" └─TableFullScan_36", "1000.00"},+ },+ want: 800,+ },+ {+ name: "limit_is_part_of_the_plan",+ plan: []planRow{+ {"Update_6", "N/A"},+ {"└─TopN_9", "20.00"},+ {" └─TableReader_17", "20.00"},+ {" └─TopN_16", "20.00"},+ {" └─Selection_15", "500.00"},+ {" └─TableFullScan_14", "1000.00"},+ },+ want: 20,+ },+ {+ name: "foreign_key_cascade_follows_the_input",+ plan: []planRow{+ {"Delete_3", "N/A"},+ {"├─TableReader_8", "9.00"},+ {"│ └─TableRangeScan_7", "9.00"},+ {"└─Foreign_Key_Cascade_4", "0.00"},+ },+ want: 9,+ },+ {+ name: "insert_set_has_no_input",+ plan: []planRow{+ {"Insert_1", "N/A"},+ },+ want: 0,+ },+ {+ // With tidb_opt_enable_non_eval_scalar_subquery, a subquery in SET is+ // planned as a second root rather than evaluated during planning.+ name: "subquery_root_is_not_an_input",+ plan: []planRow{+ {"Insert_1", "N/A"},+ {"ScalarSubQuery_36", "N/A"},+ {"└─MaxOneRow_11", "1.00"},+ {" └─Projection_12", "1.00"},+ {" └─StreamAgg_14", "1.00"},+ {" └─Limit_30", "1.00"},+ {" └─TableReader_35", "1.00"},+ {" └─Limit_34", "1.00"},+ {" └─TableFullScan_24", "1.00"},+ },+ want: 0,+ },+ {+ name: "select_root_is_rejected",+ plan: []planRow{+ {"TableReader_7", "100.00"},+ {"└─Selection_6", "100.00"},+ {" └─TableFullScan_5", "1000.00"},+ },+ wantErr: true,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, err := getAffectedRowsFromPlan(tc.plan)+ if tc.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ require.Equal(t, tc.want, common.RoundRows(got))+ })+ }+}++func TestGetAffectedRows(t *testing.T) {+ plan := []planRow{+ {"Delete_7", "N/A"},+ {"└─HashJoin_20", "0.40"},+ }+ // 0.4 joined rows change a row in each of two tables.+ got, err := getAffectedRows(plan, "DELETE a, b FROM a JOIN b ON a.id = b.id")+ require.NoError(t, err)+ require.Equal(t, int64(1), got)+}++func TestCountDMLTargets(t *testing.T) {+ for _, tc := range []struct {+ statement string+ want int+ }{+ {statement: "UPDATE t SET a = 1, b = 2 WHERE id = 1", want: 1},+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.c = 1, a.d = 2", want: 1},+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.c = 1, b.c = 1", want: 2},+ {statement: "UPDATE a AS x, b AS y SET x.c = 1, y.c = 1 WHERE x.id = y.id", want: 2},+ // Either joined table may own each unqualified column, up to the two joined tables.+ {statement: "UPDATE a JOIN b ON a.id = b.id SET c = 1, d = 2, e = 3", want: 2},+ {statement: "DELETE a, b FROM a JOIN b ON a.id = b.id", want: 2},+ {statement: "DELETE a FROM a JOIN b ON a.id = b.id", want: 1},+ {statement: "DELETE FROM t WHERE id = 1", want: 1},+ {statement: "INSERT INTO t SELECT * FROM s", want: 1},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ require.Equal(t, tc.want, countDMLTargets(tc.statement))+ })+ }+}diff --git a/backend/plugin/db/tidb/testutil_test.go b/backend/plugin/db/tidb/testutil_test.gonew file mode 100644index 0000000000..942f7e2dae--- /dev/null+++ b/backend/plugin/db/tidb/testutil_test.go@@ -0,0 +1,32 @@+package tidb++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+)++func openTestDriver(ctx context.Context, t *testing.T, container *testcontainer.Container) *Driver {+ t.Helper()++ driver := &Driver{}+ d, err := driver.Open(ctx, storepb.Engine_TIDB, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Username: "root",+ Host: container.GetHost(),+ Port: container.GetPort(),+ },+ ConnectionContext: db.ConnectionContext{},+ })+ require.NoError(t, err)++ tidbDriver, ok := d.(*Driver)+ require.True(t, ok)+ return tidbDriver+}diff --git a/backend/plugin/db/tidb/tidb_test.go b/backend/plugin/db/tidb/tidb_test.goindex 9d07428224..20c53ad3f7 100644--- a/backend/plugin/db/tidb/tidb_test.go+++ b/backend/plugin/db/tidb/tidb_test.go@@ -92,26 +92,6 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.Trequire.Equal(t, statement, commands[0].Text)}-func openTestDriver(ctx context.Context, t *testing.T, container *testcontainer.Container) *Driver {- t.Helper()-- driver := &Driver{}- d, err := driver.Open(ctx, storepb.Engine_TIDB, db.ConnectionConfig{- DataSource: &storepb.DataSource{- Type: storepb.DataSourceType_ADMIN,- Username: "root",- Host: container.GetHost(),- Port: container.GetPort(),- },- ConnectionContext: db.ConnectionContext{},- })- require.NoError(t, err)-- tidbDriver, ok := d.(*Driver)- require.True(t, ok)- return tidbDriver-}-func TestExecuteCreateIndexInTransaction(t *testing.T) {t.Parallel()ctx := context.Background()diff --git a/backend/plugin/parser/base/interface.go b/backend/plugin/parser/base/interface.goindex 0b0a74d4f2..9e7a35a029 100644--- a/backend/plugin/parser/base/interface.go+++ b/backend/plugin/parser/base/interface.go@@ -393,9 +393,39 @@ func isAllDMLImpl(engine storepb.Engine, statement string) bool {type ChangeSummary struct {ChangedResources *model.ChangedResources- SampleDMLS []string- DMLCount int- InsertCount int+ // DMLStatements holds the text of every DML statement whose rows can be estimated, in order.+ // A statement that runs after the sheet changes the search path starts with WithSearchPath's+ // prefix, which the driver splits off with SplitSearchPath and runs before the EXPLAIN.+ DMLStatements []string+ // DMLCount counts the DML statements, including those DMLStatements omits.+ DMLCount int+ // InsertCount counts the rows of INSERT ... VALUES statements.+ InsertCount int+}++// searchPathPrefix starts the statement WithSearchPath puts before a DML statement.+const searchPathPrefix = "SET LOCAL search_path TO "++// WithSearchPath returns statement preceded by a SET LOCAL search_path statement for schemas.+func WithSearchPath(statement string, schemas []string) string {+ quoted := make([]string, len(schemas))+ for i, schema := range schemas {+ quoted[i] = `"` + strings.ReplaceAll(schema, `"`, `""`) + `"`+ }+ return searchPathPrefix + strings.Join(quoted, ", ") + ";\n" + statement+}++// SplitSearchPath returns the SET LOCAL search_path statement that WithSearchPath put before+// statement, or "" when there is none, and the statement after it.+func SplitSearchPath(statement string) (string, string) {+ if !strings.HasPrefix(statement, searchPathPrefix) {+ return "", statement+ }+ setup, rest, ok := strings.Cut(statement, ";\n")+ if !ok {+ return "", statement+ }+ return setup, rest}// REFACTOR(zp): Put it here to avoid circular import for now.diff --git a/backend/plugin/parser/base/interface_test.go b/backend/plugin/parser/base/interface_test.goindex 1001937c76..660dd03382 100644--- a/backend/plugin/parser/base/interface_test.go+++ b/backend/plugin/parser/base/interface_test.go@@ -1,6 +1,23 @@package base-import "testing"+import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestWithSearchPath(t *testing.T) {+ statement := WithSearchPath("DELETE FROM t;", []string{"app", `we"ird`})+ require.Equal(t, "SET LOCAL search_path TO \"app\", \"we\"\"ird\";\nDELETE FROM t;", statement)++ setup, rest := SplitSearchPath(statement)+ require.Equal(t, `SET LOCAL search_path TO "app", "we""ird"`, setup)+ require.Equal(t, "DELETE FROM t;", rest)++ setup, rest = SplitSearchPath("DELETE FROM t;")+ require.Empty(t, setup)+ require.Equal(t, "DELETE FROM t;", rest)+}func TestTSQLRecognizeExplainType(t *testing.T) {testCases := []struct {diff --git a/backend/plugin/parser/cockroachdb/resource_change.go b/backend/plugin/parser/cockroachdb/resource_change.gonew file mode 100644index 0000000000..e850bf00f2--- /dev/null+++ b/backend/plugin/parser/cockroachdb/resource_change.go@@ -0,0 +1,477 @@+package cockroachdb++import (+ "slices"+ "strings"++ "github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++func init() {+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_COCKROACHDB, extractChangedResources)+}++// extractChangedResources resolves an unqualified table to the current database and to a schema of+// the search path: currentSchema, or else the search path synced in dbMetadata, or "public", until+// the statements SET another. Like the PostgreSQL extractor, an existing table resolves to the first+// schema that has it in dbMetadata, and a new table to the first schema.+func extractChangedResources(database string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {+ defaultSearchPath := []string{currentSchema}+ if currentSchema == "" {+ defaultSearchPath = []string{"public"}+ if dbMetadata != nil && len(dbMetadata.GetSearchPath()) > 0 {+ defaultSearchPath = dbMetadata.GetSearchPath()+ }+ }+ searchPath := defaultSearchPath+ // sessionSearchPath leaves out SET LOCAL, which lasts until the transaction ends.+ sessionSearchPath := searchPath+ summary := &base.ChangeSummary{+ ChangedResources: model.NewChangedResources(dbMetadata),+ }+ schemaOf := func(table string) string {+ if dbMetadata == nil {+ return searchPath[0]+ }+ for _, schemaName := range searchPath {+ schema := dbMetadata.GetSchemaMetadata(schemaName)+ if schema != nil && (schema.GetTable(table) != nil || schema.GetView(table) != nil || schema.GetMaterializedView(table) != nil) {+ return schemaName+ }+ }+ return searchPath[0]+ }+ addTable := func(name *tree.TableName, affectedTable bool) {+ db, schema, table := resolveTableName(name, database, schemaOf(name.Table()))+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, affectedTable)+ }+ // addIndexTable records the table of an index, which dbMetadata supplies when the name omits it.+ addIndexTable := func(index *tree.TableIndexName) {+ if index.Table.ObjectName != "" {+ addTable(&index.Table, false)+ return+ }+ // dbMetadata describes only the current database, so an index of another is recorded by its database.+ if index.Table.ExplicitCatalog && index.Table.Catalog() != database {+ summary.ChangedResources.AddDatabase(index.Table.Catalog())+ return+ }+ if dbMetadata == nil {+ return+ }+ schemas := searchPath+ if index.Table.ExplicitSchema {+ schemas = []string{index.Table.Schema()}+ }+ schema, indexMetadata := dbMetadata.SearchIndex(schemas, string(index.Index))+ if indexMetadata == nil {+ return+ }+ db, _, _ := resolveTableName(&index.Table, database, schema)+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: indexMetadata.GetTableProto().GetName()}, false)+ }+ // addMutations records the target of each mutation in the statement and returns their number.+ addMutations := func(stmt tree.Statement) int {+ targets := getMutationTargets(stmt)+ for _, target := range targets {+ if name, ok := getTableName(target); ok {+ addTable(name, false)+ }+ }+ return len(targets)+ }+ addDML := func(text string) {+ summary.DMLCount+++ text = strings.TrimSpace(text)+ // The EXPLAIN connection starts with the default search path, so a changed one is replayed.+ if !slices.Equal(searchPath, defaultSearchPath) {+ text = base.WithSearchPath(text, searchPath)+ }+ summary.DMLStatements = append(summary.DMLStatements, text)+ }++ for _, ast := range asts {+ crdbAST, ok := ast.(*AST)+ if !ok {+ return nil, errors.New("expected CockroachDB AST")+ }+ stmt, text := crdbAST.Stmt.AST, crdbAST.Stmt.SQL+ // EXPLAIN ANALYZE executes the statement it explains.+ if explain, ok := stmt.(*tree.ExplainAnalyze); ok {+ stmt, text = explain.Statement, tree.AsString(explain.Statement)+ }+ switch n := stmt.(type) {+ case *tree.Insert:+ mutationCount := addMutations(n)+ // Nested mutations make the whole statement a sample, whose estimate includes the VALUES rows.+ if rows, ok := getInsertValuesRowCount(n); ok && mutationCount == 1 {+ summary.InsertCount += rows+ } else {+ addDML(text)+ }+ case *tree.Update, *tree.Delete:+ addMutations(n)+ addDML(text)+ case *tree.Select:+ if addMutations(n) > 0 {+ addDML(text)+ }+ case *tree.CreateTable:+ db, schema, table := resolveTableName(&n.Table, database, searchPath[0])+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)+ if n.AsSource != nil && addMutations(n.AsSource) > 0 {+ addDML(text)+ }+ case *tree.CreateView:+ if n.Materialized {+ db, schema, table := resolveTableName(&n.Name, database, searchPath[0])+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)+ }+ case *tree.Import:+ // The plan cannot estimate the rows an import loads, so, as for LOAD DATA in the MySQL family,+ // only its target is recorded.+ if n.Into && n.Table != nil {+ addTable(n.Table, false)+ }+ case *tree.AlterTable:+ name := n.Table.ToTableName()+ addTable(&name, true)+ case *tree.AlterTableLocality:+ name := n.Name.ToTableName()+ addTable(&name, true)+ case *tree.AlterTableSetSchema:+ if !n.IsSequence && (!n.IsView || n.IsMaterialized) {+ name := n.Name.ToTableName()+ addTable(&name, true)+ }+ case *tree.AlterTableOwner:+ if !n.IsSequence && (!n.IsView || n.IsMaterialized) {+ name := n.Name.ToTableName()+ addTable(&name, true)+ }+ case *tree.DropTable:+ for i := range n.Names {+ addTable(&n.Names[i], true)+ }+ case *tree.DropView:+ // A materialized view holds data, so its drop is recorded like a table's.+ if n.IsMaterialized {+ for i := range n.Names {+ addTable(&n.Names[i], true)+ }+ }+ case *tree.Truncate:+ for i := range n.Tables {+ addTable(&n.Tables[i], true)+ }+ case *tree.RenameTable:+ if n.IsSequence || (n.IsView && !n.IsMaterialized) {+ continue+ }+ name := n.Name.ToTableName()+ db, schema, table := resolveTableName(&name, database, schemaOf(name.Table()))+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, true)+ // An unqualified new name stays in the schema of the renamed table.+ newName := n.NewName.ToTableName()+ newDB, newSchema, newTable := resolveTableName(&newName, db, schema)+ summary.ChangedResources.AddTable(newDB, newSchema, &storepb.ChangedResourceTable{Name: newTable}, false)+ case *tree.CreateIndex:+ addTable(&n.Table, false)+ case *tree.DropIndex:+ for _, index := range n.IndexList {+ addIndexTable(index)+ }+ case *tree.AlterIndex:+ addIndexTable(&n.Index)+ case *tree.AlterIndexVisible:+ addIndexTable(&n.Index)+ case *tree.RenameIndex:+ addIndexTable(n.Index)+ case *tree.SetZoneConfig:+ switch {+ case n.TableOrIndex.Index != "":+ addIndexTable(&n.TableOrIndex)+ case n.TableOrIndex.Table.ObjectName != "":+ addTable(&n.TableOrIndex.Table, true)+ default:+ }+ case *tree.Discard:+ if n.Mode == tree.DiscardModeAll {+ searchPath, sessionSearchPath = defaultSearchPath, defaultSearchPath+ }+ case *tree.CommitTransaction, *tree.RollbackTransaction:+ // Unlike PostgreSQL, CockroachDB keeps a SET made in a transaction that rolls back.+ searchPath = sessionSearchPath+ case *tree.SetVar:+ if n.ResetAll || strings.EqualFold(n.Name, "search_path") {+ searchPath = getSearchPath(n, defaultSearchPath)+ if !n.Local {+ sessionSearchPath = searchPath+ }+ }+ default:+ }+ }+ return summary, nil+}++// getSearchPath returns the schemas a SET search_path names, skipping "$user", or defaultSearchPath+// when it names none or resets the search path.+func getSearchPath(set *tree.SetVar, defaultSearchPath []string) []string {+ if set.ResetAll {+ return defaultSearchPath+ }+ var searchPath []string+ for _, value := range set.Values {+ var schema string+ switch v := value.(type) {+ case *tree.UnresolvedName:+ schema = v.Parts[0]+ case *tree.StrVal:+ schema = v.RawString()+ default:+ return defaultSearchPath+ }+ if schema != "$user" {+ searchPath = append(searchPath, schema)+ }+ }+ if len(searchPath) == 0 {+ return defaultSearchPath+ }+ return searchPath+}++func resolveTableName(name *tree.TableName, defaultDatabase string, defaultSchema string) (string, string, string) {+ database, schema := defaultDatabase, defaultSchema+ if name.ExplicitCatalog {+ database = name.Catalog()+ }+ if name.ExplicitSchema {+ schema = name.Schema()+ }+ return database, schema, name.Table()+}++// getTableName returns the table name of a mutation target, which is false for a numeric table+// reference.+func getTableName(target tree.TableExpr) (*tree.TableName, bool) {+ if aliased, ok := target.(*tree.AliasedTableExpr); ok {+ target = aliased.Expr+ }+ name, ok := target.(*tree.TableName)+ return name, ok+}++// getInsertValuesRowCount returns the rows of INSERT ... VALUES and INSERT ... DEFAULT VALUES; ok is+// false when a query supplies the rows.+func getInsertValuesRowCount(insert *tree.Insert) (int, bool) {+ if insert.Rows == nil || insert.Rows.Select == nil {+ return 1, true+ }+ rows := insert.Rows.Select+ for {+ paren, ok := rows.(*tree.ParenSelect)+ if !ok || paren.Select == nil {+ break+ }+ rows = paren.Select.Select+ }+ if values, ok := rows.(*tree.ValuesClause); ok {+ return len(values.Rows), true+ }+ return 0, false+}++// getMutationTargets returns the target of every INSERT, UPSERT, UPDATE, and DELETE in stmt: stmt+// itself, its CTEs, and statement sources such as [DELETE FROM t RETURNING id] at any depth.+func getMutationTargets(stmt tree.Statement) []tree.TableExpr {+ c := &mutationCollector{}+ c.statement(stmt)+ return c.targets+}++// getMutationTypes returns the statement type of each mutation getMutationTargets finds, in order.+func getMutationTypes(stmt tree.Statement) []storepb.StatementType {+ c := &mutationCollector{}+ c.statement(stmt)+ return c.types+}++type mutationCollector struct {+ targets []tree.TableExpr+ types []storepb.StatementType+}++func (c *mutationCollector) statement(stmt tree.Statement) {+ switch n := stmt.(type) {+ case *tree.Insert:+ c.targets = append(c.targets, n.Table)+ c.types = append(c.types, storepb.StatementType_INSERT)+ c.with(n.With)+ c.selectQuery(n.Rows)+ if n.OnConflict != nil {+ c.updateExprs(n.OnConflict.Exprs)+ c.where(n.OnConflict.Where)+ c.expr(n.OnConflict.ArbiterPredicate)+ }+ c.returning(n.Returning)+ case *tree.Update:+ c.targets = append(c.targets, n.Table)+ c.types = append(c.types, storepb.StatementType_UPDATE)+ c.with(n.With)+ c.updateExprs(n.Exprs)+ c.tableExprs(n.From)+ c.where(n.Where)+ c.orderBy(n.OrderBy)+ c.limit(n.Limit)+ c.returning(n.Returning)+ case *tree.Delete:+ c.targets = append(c.targets, n.Table)+ c.types = append(c.types, storepb.StatementType_DELETE)+ c.with(n.With)+ c.tableExprs(n.Using)+ c.where(n.Where)+ c.orderBy(n.OrderBy)+ c.limit(n.Limit)+ c.returning(n.Returning)+ case *tree.Select:+ c.selectQuery(n)+ case tree.SelectStatement:+ c.selectStatement(n)+ default:+ }+}++func (c *mutationCollector) selectQuery(sel *tree.Select) {+ if sel == nil {+ return+ }+ c.with(sel.With)+ c.selectStatement(sel.Select)+ c.orderBy(sel.OrderBy)+ c.limit(sel.Limit)+}++func (c *mutationCollector) selectStatement(stmt tree.SelectStatement) {+ switch n := stmt.(type) {+ case *tree.ParenSelect:+ c.selectQuery(n.Select)+ case *tree.SelectClause:+ for _, selectExpr := range n.Exprs {+ c.expr(selectExpr.Expr)+ }+ c.tableExprs(n.From.Tables)+ c.where(n.Where)+ c.exprs(n.GroupBy)+ c.where(n.Having)+ c.exprs(n.DistinctOn)+ for _, window := range n.Window {+ c.exprs(window.Partitions)+ c.orderBy(window.OrderBy)+ }+ case *tree.UnionClause:+ c.selectQuery(n.Left)+ c.selectQuery(n.Right)+ case *tree.ValuesClause:+ for _, row := range n.Rows {+ c.exprs(row)+ }+ default:+ }+}++func (c *mutationCollector) with(with *tree.With) {+ if with == nil {+ return+ }+ for _, cte := range with.CTEList {+ c.statement(cte.Stmt)+ }+}++func (c *mutationCollector) tableExprs(exprs tree.TableExprs) {+ for _, expr := range exprs {+ c.tableExpr(expr)+ }+}++func (c *mutationCollector) tableExpr(expr tree.TableExpr) {+ switch n := expr.(type) {+ case *tree.AliasedTableExpr:+ c.tableExpr(n.Expr)+ case *tree.ParenTableExpr:+ c.tableExpr(n.Expr)+ case *tree.JoinTableExpr:+ c.tableExpr(n.Left)+ c.tableExpr(n.Right)+ if cond, ok := n.Cond.(*tree.OnJoinCond); ok {+ c.expr(cond.Expr)+ }+ case *tree.RowsFromExpr:+ c.exprs(n.Items)+ case *tree.Subquery:+ c.selectStatement(n.Select)+ case *tree.StatementSource:+ c.statement(n.Statement)+ default:+ }+}++func (c *mutationCollector) expr(expr tree.Expr) {+ if expr == nil {+ return+ }+ // The visit function never fails, so neither does the visit.+ _, _ = tree.SimpleVisit(expr, func(e tree.Expr) (bool, tree.Expr, error) {+ if subquery, ok := e.(*tree.Subquery); ok {+ c.selectStatement(subquery.Select)+ return false, e, nil+ }+ return true, e, nil+ })+}++func (c *mutationCollector) exprs(exprs []tree.Expr) {+ for _, expr := range exprs {+ c.expr(expr)+ }+}++func (c *mutationCollector) where(where *tree.Where) {+ if where != nil {+ c.expr(where.Expr)+ }+}++func (c *mutationCollector) orderBy(orderBy tree.OrderBy) {+ for _, order := range orderBy {+ c.expr(order.Expr)+ }+}++func (c *mutationCollector) limit(limit *tree.Limit) {+ if limit != nil {+ c.expr(limit.Count)+ c.expr(limit.Offset)+ }+}++func (c *mutationCollector) updateExprs(exprs tree.UpdateExprs) {+ for _, updateExpr := range exprs {+ c.expr(updateExpr.Expr)+ }+}++func (c *mutationCollector) returning(returning tree.ReturningClause) {+ if exprs, ok := returning.(*tree.ReturningExprs); ok {+ for _, selectExpr := range *exprs {+ c.expr(selectExpr.Expr)+ }+ }+}diff --git a/backend/plugin/parser/cockroachdb/resource_change_test.go b/backend/plugin/parser/cockroachdb/resource_change_test.gonew file mode 100644index 0000000000..cf18e447ed--- /dev/null+++ b/backend/plugin/parser/cockroachdb/resource_change_test.go@@ -0,0 +1,401 @@+package cockroachdb++import (+ "fmt"+ "slices"+ "strings"+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"++ // Registers the statement splitter that CockroachDB parsing uses.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/pg"+)++func TestExtractChangedResources(t *testing.T) {+ const statement = `CREATE TABLE t1 (c1 INT);+DROP TABLE t1;+ALTER TABLE t1 ADD COLUMN c2 INT;+ALTER TABLE t1 RENAME TO t2;+INSERT INTO t1 (c1) VALUES (1), (5);+UPDATE t1+SET c1 = 5+WHERE c1 = 1;+DELETE FROM t1; -- trailing comment+`+ want := model.NewChangedResources(nil /* dbMetadata */)+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t1"}, true)+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t2"}, false)++ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, &base.ChangeSummary{+ ChangedResources: want,+ DMLCount: 2,+ DMLStatements: []string{"UPDATE t1\nSET c1 = 5\nWHERE c1 = 1", "DELETE FROM t1"},+ InsertCount: 2,+ }, got)+}++func TestExtractChangedResourcesDMLCounts(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ wantDMLCount int+ wantInsertCount int+ wantTables []string+ }{+ {+ name: "insert values",+ statement: `INSERT INTO t VALUES (1), (2), (3)`,+ wantInsertCount: 3,+ wantTables: []string{"t"},+ },+ {+ name: "parenthesized insert values",+ statement: `INSERT INTO t (VALUES (1), (2))`,+ wantInsertCount: 2,+ wantTables: []string{"t"},+ },+ {+ name: "upsert values",+ statement: `UPSERT INTO t VALUES (1), (2)`,+ wantInsertCount: 2,+ wantTables: []string{"t"},+ },+ {+ name: "insert values on conflict",+ statement: `INSERT INTO t VALUES (1), (2) ON CONFLICT (id) DO UPDATE SET c = excluded.c`,+ wantInsertCount: 2,+ wantTables: []string{"t"},+ },+ {+ name: "insert default values",+ statement: `INSERT INTO t DEFAULT VALUES`,+ wantInsertCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "insert select",+ statement: `INSERT INTO t SELECT * FROM t2 ON CONFLICT DO NOTHING`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "upsert select",+ statement: `UPSERT INTO t SELECT * FROM t2`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "insert select with read-only CTE",+ statement: `WITH x AS (SELECT id FROM t2) INSERT INTO t SELECT id FROM x`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "update",+ statement: `UPDATE t SET c = 1 FROM t2 WHERE t.id = t2.id`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "delete",+ statement: `DELETE FROM t USING t2 WHERE t.id = t2.id ORDER BY t.id LIMIT 10`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "data-modifying CTE feeding insert",+ statement: `WITH d AS (DELETE FROM t WHERE id > 0 RETURNING *) INSERT INTO t2 SELECT * FROM d`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "data-modifying CTE feeding insert values",+ statement: `WITH i AS (INSERT INTO t VALUES (1) RETURNING id) INSERT INTO t2 VALUES (2), (3)`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "data-modifying CTE in select",+ statement: `WITH d AS (DELETE FROM t RETURNING 1) SELECT count(*) FROM d`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "data-modifying CTEs feeding delete",+ statement: `WITH u AS (UPDATE t SET c = 1 RETURNING id), i AS (UPSERT INTO t3 SELECT * FROM t2 RETURNING id) DELETE FROM t2 WHERE id IN (SELECT id FROM u)`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2", "t3"},+ },+ {+ name: "statement source in select",+ statement: `SELECT * FROM [DELETE FROM t WHERE c < 10 RETURNING id]`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "statement source in scalar subquery of insert values",+ statement: `INSERT INTO t2 VALUES ((SELECT max(id) FROM [DELETE FROM t RETURNING id]))`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "statement source in update subquery",+ statement: `UPDATE t SET c = 1 WHERE id IN (SELECT id FROM [DELETE FROM t2 WHERE c < 3 RETURNING id])`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "statement source joined in delete using",+ statement: `DELETE FROM t USING t3 JOIN [UPDATE t2 SET c = 1 RETURNING id] AS u ON t3.id = u.id WHERE t.id = t3.id`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "statement source in create table as",+ statement: `CREATE TABLE t_moved AS SELECT * FROM [DELETE FROM t RETURNING *]`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t_moved"},+ },+ {+ name: "create table as",+ statement: `CREATE TABLE t_copy AS SELECT * FROM t`,+ wantTables: []string{"t_copy"},+ },+ {+ name: "read-only CTE in select",+ statement: `WITH x AS (SELECT id FROM t) SELECT * FROM x WHERE id IN (SELECT id FROM [SHOW TABLES])`,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ want := model.NewChangedResources(nil /* dbMetadata */)+ for _, table := range tc.wantTables {+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)+ }+ var wantDMLStatements []string+ if tc.wantDMLCount > 0 {+ wantDMLStatements = []string{tc.statement}+ }++ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, tc.statement), tc.statement)+ require.NoError(t, err)+ require.Equal(t, &base.ChangeSummary{+ ChangedResources: want,+ DMLCount: tc.wantDMLCount,+ DMLStatements: wantDMLStatements,+ InsertCount: tc.wantInsertCount,+ }, got)+ })+ }+}++func TestExtractChangedResourcesFollowsSearchPath(t *testing.T) {+ const statement = `DELETE FROM t1;+SET search_path = app, public;+DELETE FROM t2;+SET search_path TO '$user', "Reports";+DELETE FROM t3;+RESET search_path;+DELETE FROM t4;`+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, []string{"db.Reports.t3", "db.app.t2", "db.public.t1", "db.public.t4"}, getTableNames(got.ChangedResources))+ // A statement after the search path changes carries it for its EXPLAIN.+ require.Equal(t, []string{+ "DELETE FROM t1",+ "SET LOCAL search_path TO \"app\", \"public\";\nDELETE FROM t2",+ "SET LOCAL search_path TO \"Reports\";\nDELETE FROM t3",+ "DELETE FROM t4",+ }, got.DMLStatements)++ t.Run("SET LOCAL lasts until the transaction ends, SET outlasts ROLLBACK, and DISCARD ALL resets", func(t *testing.T) {+ const statement = `BEGIN;+SET LOCAL search_path = app;+DELETE FROM t1;+COMMIT;+DELETE FROM t2;+BEGIN;+SET search_path = app;+SET LOCAL search_path = other;+ROLLBACK;+DELETE FROM t3;+DISCARD ALL;+DELETE FROM t4;`+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, []string{"db.app.t1", "db.app.t3", "db.public.t2", "db.public.t4"}, getTableNames(got.ChangedResources))+ })++ t.Run("the synced search path applies without a current schema", func(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ SearchPath: `"$user", app, public`,+ Schemas: []*metadatapb.SchemaMetadata{+ {Name: "app", Tables: []*metadatapb.TableMetadata{{Name: "t"}}},+ {Name: "public", Tables: []*metadatapb.TableMetadata{{Name: "t2"}}},+ },+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)+ const statement = `DELETE FROM t; DELETE FROM t2; DROP MATERIALIZED VIEW mv; DROP VIEW v;`+ got, err := extractChangedResources("db", "", dbMetadata, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, []string{"db.app.mv", "db.app.t", "db.public.t2"}, getTableNames(got.ChangedResources))+ })++ t.Run("existing tables resolve to the first schema that has them", func(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {Name: "public", Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 1000}}},+ {Name: "empty"},+ },+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)+ const statement = `SET search_path = empty, public;+DELETE FROM t;+CREATE TABLE t2 (id INT PRIMARY KEY);`+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, []string{"db.empty.t2", "db.public.t"}, getTableNames(got.ChangedResources))+ })+}++func TestExtractChangedResourcesNames(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ currentSchema string+ want string+ }{+ {name: "unqualified defaults to public", statement: `UPDATE t SET c = 1`, want: "db.public.t"},+ {name: "unqualified uses the current schema", statement: `UPDATE t SET c = 1`, currentSchema: "app", want: "db.app.t"},+ {name: "schema qualified with alias", statement: `INSERT INTO myschema.t AS x VALUES (1)`, currentSchema: "app", want: "db.myschema.t"},+ {name: "database qualified", statement: `DELETE FROM otherdb.myschema.t`, want: "otherdb.myschema.t"},+ {name: "quoted identifiers keep their case", statement: `UPDATE "MySchema"."T" SET c = 1`, want: "db.MySchema.T"},+ {name: "unquoted identifiers are lowercased", statement: `TRUNCATE MySchema.T`, want: "db.myschema.t"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, err := extractChangedResources("db", tc.currentSchema, nil /* dbMetadata */, parseASTs(t, tc.statement), tc.statement)+ require.NoError(t, err)+ require.Equal(t, []string{tc.want}, getTableNames(got.ChangedResources))+ })+ }+}++func TestExtractChangedResourcesDDL(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {Name: "t", RowCount: 10},+ {Name: "t2", RowCount: 1000, Indexes: []*metadatapb.IndexMetadata{{Name: "t2_c_idx"}}},+ },+ },+ {+ Name: "app",+ Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 100_000, Indexes: []*metadatapb.IndexMetadata{{Name: "t_c_idx"}}}},+ },+ },+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)+ for _, tc := range []struct {+ statement string+ wantTables []string+ wantAffectedRows int64+ }{+ {statement: `CREATE TABLE t3 (id INT PRIMARY KEY)`, wantTables: []string{"db.public.t3"}},+ {statement: `CREATE INDEX idx ON t2 (c)`, wantTables: []string{"db.public.t2"}},+ {statement: `CREATE MATERIALIZED VIEW mv AS SELECT * FROM t`, wantTables: []string{"db.public.mv"}},+ {statement: `CREATE VIEW v AS SELECT * FROM t`},+ {statement: `IMPORT INTO t2 (c) CSV DATA ('nodelocal://1/t2.csv')`, wantTables: []string{"db.public.t2"}},+ {statement: `ALTER TABLE t2 CONFIGURE ZONE USING num_replicas = 5`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},+ {statement: `ALTER INDEX t2_c_idx CONFIGURE ZONE USING num_replicas = 5`, wantTables: []string{"db.public.t2"}},+ {statement: `ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5`},+ {statement: `DROP INDEX t2@t2_c_idx, t2_c_idx, app.t_c_idx, missing_idx`, wantTables: []string{"db.app.t", "db.public.t2"}},+ {statement: `DROP INDEX db.public.t2_c_idx`, wantTables: []string{"db.public.t2"}},+ {statement: `ALTER INDEX app.t@t_c_idx PARTITION BY NOTHING`, wantTables: []string{"db.app.t"}},+ {statement: `ALTER INDEX t2_c_idx NOT VISIBLE`, wantTables: []string{"db.public.t2"}},+ {statement: `ALTER INDEX t2_c_idx RENAME TO t2_idx`, wantTables: []string{"db.public.t2"}},+ {statement: `ALTER TABLE t ADD COLUMN c2 INT, DROP COLUMN c`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE t RENAME COLUMN c TO c2`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE t SET LOCALITY REGIONAL BY ROW`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE t SET SCHEMA app`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE t OWNER TO u`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},+ {statement: `ALTER VIEW t2 SET SCHEMA app`},+ {statement: `ALTER MATERIALIZED VIEW t2 SET SCHEMA app`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},+ {statement: `ALTER MATERIALIZED VIEW t2 OWNER TO u`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},+ {statement: `ALTER TABLE t RENAME TO t_new`, wantTables: []string{"db.public.t", "db.public.t_new"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE app.t RENAME TO t_new`, wantTables: []string{"db.app.t", "db.app.t_new"}, wantAffectedRows: 100_000},+ {statement: `ALTER MATERIALIZED VIEW t2 RENAME TO mv`, wantTables: []string{"db.public.mv", "db.public.t2"}, wantAffectedRows: 1000},+ {statement: `ALTER VIEW t2 RENAME TO v`},+ {statement: `ALTER SEQUENCE t2 RENAME TO s`},+ {statement: `DROP TABLE t, app.t`, wantTables: []string{"db.app.t", "db.public.t"}, wantAffectedRows: 100_010},+ {statement: `TRUNCATE t2`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, tc.statement), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.wantTables, getTableNames(got.ChangedResources))+ require.Equal(t, tc.wantAffectedRows, got.ChangedResources.CountAffectedTableRows())+ require.Zero(t, got.DMLCount)+ })+ }++ t.Run("an index of another database is recorded by its database", func(t *testing.T) {+ const statement = `DROP INDEX otherdb.public.t2_c_idx`+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Empty(t, getTableNames(got.ChangedResources))+ require.Equal(t, []string{"otherdb"}, got.ChangedResources.GetDatabaseOnlyTargets())+ })+}++func TestExtractChangedResourcesExplainAnalyze(t *testing.T) {+ const statement = `EXPLAIN ANALYZE (VERBOSE) DELETE FROM t WHERE id = 1;+EXPLAIN ANALYZE UPSERT INTO t2 SELECT * FROM t;+EXPLAIN UPDATE t3 SET c = 1;`+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, []string{"db.public.t", "db.public.t2"}, getTableNames(got.ChangedResources))+ require.Equal(t, []string{"DELETE FROM t WHERE id = 1", "UPSERT INTO t2 SELECT * FROM t"}, got.DMLStatements)+}++func TestExtractChangedResourcesListsEveryDMLStatement(t *testing.T) {+ var statements []string+ for i := range 12 {+ statements = append(statements, fmt.Sprintf("DELETE FROM t WHERE id = %d", i))+ }+ statement := strings.Join(statements, ";\n")++ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)+ require.NoError(t, err)+ require.Equal(t, 12, got.DMLCount)+ require.Equal(t, statements, got.DMLStatements)+}++func parseASTs(t *testing.T, statement string) []base.AST {+ t.Helper()+ stmts, err := base.ParseStatements(storepb.Engine_COCKROACHDB, statement)+ require.NoError(t, err)+ return base.ExtractASTs(stmts)+}++// getTableNames returns the changed tables as sorted database.schema.table names.+func getTableNames(changedResources *model.ChangedResources) []string {+ var names []string+ for _, database := range changedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ names = append(names, database.GetName()+"."+schema.GetName()+"."+table.GetName())+ }+ }+ }+ slices.Sort(names)+ return names+}diff --git a/backend/plugin/parser/cockroachdb/statement_type.go b/backend/plugin/parser/cockroachdb/statement_type.gonew file mode 100644index 0000000000..4a997696e6--- /dev/null+++ b/backend/plugin/parser/cockroachdb/statement_type.go@@ -0,0 +1,175 @@+package cockroachdb++import (+ "slices"++ "github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterGetStatementTypes(storepb.Engine_COCKROACHDB, GetStatementTypes)+}++// GetStatementTypes returns the types of the statements that have one, in order, each followed by+// the types of the mutations in its CTEs and statement sources, each type once per statement.+func GetStatementTypes(asts []base.AST) ([]storepb.StatementType, error) {+ var statementTypes []storepb.StatementType+ for _, ast := range asts {+ crdbAST, ok := ast.(*AST)+ if !ok {+ return nil, errors.New("expected CockroachDB AST")+ }+ var types []storepb.StatementType+ add := func(statementType storepb.StatementType) {+ if statementType != storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED && !slices.Contains(types, statementType) {+ types = append(types, statementType)+ }+ }+ stmt := crdbAST.Stmt.AST+ // EXPLAIN ANALYZE executes the statement it explains.+ if explain, ok := stmt.(*tree.ExplainAnalyze); ok {+ stmt = explain.Statement+ }+ add(getStatementType(stmt))+ if createTable, ok := stmt.(*tree.CreateTable); ok && createTable.AsSource != nil {+ stmt = createTable.AsSource+ }+ for _, mutationType := range getMutationTypes(stmt) {+ add(mutationType)+ }+ statementTypes = append(statementTypes, types...)+ }+ return statementTypes, nil+}++func getStatementType(stmt tree.Statement) storepb.StatementType {+ switch n := stmt.(type) {+ case *tree.CreateDatabase:+ return storepb.StatementType_CREATE_DATABASE+ case *tree.CreateTable:+ return storepb.StatementType_CREATE_TABLE+ case *tree.CreateView:+ return storepb.StatementType_CREATE_VIEW+ case *tree.CreateIndex:+ return storepb.StatementType_CREATE_INDEX+ case *tree.CreateSequence:+ return storepb.StatementType_CREATE_SEQUENCE+ case *tree.CreateSchema:+ return storepb.StatementType_CREATE_SCHEMA+ case *tree.CreateRoutine:+ // Procedures are functions here, as PostgreSQL classifies CREATE PROCEDURE.+ return storepb.StatementType_CREATE_FUNCTION+ case *tree.CreateTrigger:+ return storepb.StatementType_CREATE_TRIGGER+ case *tree.CreateExtension:+ return storepb.StatementType_CREATE_EXTENSION+ case *tree.CreateType:+ return storepb.StatementType_CREATE_TYPE++ case *tree.DropDatabase:+ return storepb.StatementType_DROP_DATABASE+ case *tree.DropTable:+ return storepb.StatementType_DROP_TABLE+ case *tree.DropView:+ // A materialized view holds data, so dropping one is as risky as dropping a table.+ if n.IsMaterialized {+ return storepb.StatementType_DROP_TABLE+ }+ return storepb.StatementType_DROP_VIEW+ case *tree.DropIndex:+ return storepb.StatementType_DROP_INDEX+ case *tree.DropSequence:+ return storepb.StatementType_DROP_SEQUENCE+ case *tree.DropSchema:+ return storepb.StatementType_DROP_SCHEMA+ case *tree.DropType:+ return storepb.StatementType_DROP_TYPE+ case *tree.DropTrigger:+ return storepb.StatementType_DROP_TRIGGER+ case *tree.DropRoutine:+ return storepb.StatementType_DROP_FUNCTION+ case *tree.Truncate:+ return storepb.StatementType_TRUNCATE++ case *tree.RenameDatabase, *tree.AlterDatabaseOwner, *tree.ReparentDatabase,+ *tree.AlterDatabaseAddRegion, *tree.AlterDatabaseDropRegion, *tree.AlterDatabasePrimaryRegion,+ *tree.AlterDatabaseSurvivalGoal, *tree.AlterDatabasePlacement, *tree.AlterDatabaseAddSuperRegion,+ *tree.AlterDatabaseDropSuperRegion, *tree.AlterDatabaseAlterSuperRegion, *tree.AlterDatabaseSecondaryRegion,+ *tree.AlterDatabaseDropSecondaryRegion, *tree.AlterDatabaseSetZoneConfigExtension:+ return storepb.StatementType_ALTER_DATABASE+ case *tree.SetZoneConfig:+ switch {+ case n.Database != "":+ return storepb.StatementType_ALTER_DATABASE+ case n.TableOrIndex.Index != "":+ return storepb.StatementType_ALTER_INDEX+ case n.TableOrIndex.Table.ObjectName != "":+ return storepb.StatementType_ALTER_TABLE+ default:+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED+ }+ case *tree.AlterRoleSet:+ // ALTER DATABASE d SET parses as ALTER ROLE ALL IN DATABASE d SET, which sets the same defaults.+ if n.AllRoles && n.DatabaseName != "" {+ return storepb.StatementType_ALTER_DATABASE+ }+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED+ case *tree.AlterTable, *tree.AlterTableLocality:+ return storepb.StatementType_ALTER_TABLE+ case *tree.AlterTableSetSchema:+ return getRelationAlterType(n.IsView && !n.IsMaterialized, n.IsSequence)+ case *tree.AlterTableOwner:+ return getRelationAlterType(n.IsView && !n.IsMaterialized, n.IsSequence)+ case *tree.AlterIndex, *tree.AlterIndexVisible:+ return storepb.StatementType_ALTER_INDEX+ case *tree.AlterSequence:+ return storepb.StatementType_ALTER_SEQUENCE+ case *tree.AlterType:+ return storepb.StatementType_ALTER_TYPE++ case *tree.RenameTable:+ switch {+ case n.IsSequence:+ return storepb.StatementType_RENAME_SEQUENCE+ case n.IsView && !n.IsMaterialized:+ return storepb.StatementType_ALTER_VIEW+ default:+ return storepb.StatementType_ALTER_TABLE+ }+ case *tree.RenameIndex:+ return storepb.StatementType_RENAME_INDEX+ case *tree.AlterSchema:+ if _, ok := n.Cmd.(*tree.AlterSchemaRename); ok {+ return storepb.StatementType_RENAME_SCHEMA+ }+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED++ case *tree.CommentOnColumn, *tree.CommentOnConstraint, *tree.CommentOnDatabase, *tree.CommentOnIndex,+ *tree.CommentOnSchema, *tree.CommentOnTable, *tree.CommentOnType:+ return storepb.StatementType_COMMENT++ case *tree.Insert:+ return storepb.StatementType_INSERT+ case *tree.Update:+ return storepb.StatementType_UPDATE+ case *tree.Delete:+ return storepb.StatementType_DELETE+ default:+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED+ }+}++func getRelationAlterType(isView bool, isSequence bool) storepb.StatementType {+ switch {+ case isSequence:+ return storepb.StatementType_ALTER_SEQUENCE+ case isView:+ return storepb.StatementType_ALTER_VIEW+ default:+ return storepb.StatementType_ALTER_TABLE+ }+}diff --git a/backend/plugin/parser/cockroachdb/statement_type_test.go b/backend/plugin/parser/cockroachdb/statement_type_test.gonew file mode 100644index 0000000000..f3770ee801--- /dev/null+++ b/backend/plugin/parser/cockroachdb/statement_type_test.go@@ -0,0 +1,96 @@+package cockroachdb++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestGetStatementTypes(t *testing.T) {+ for _, tc := range []struct {+ statement string+ want []storepb.StatementType+ }{+ {statement: `CREATE DATABASE d`, want: []storepb.StatementType{storepb.StatementType_CREATE_DATABASE}},+ {statement: `CREATE TABLE t (id INT PRIMARY KEY)`, want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE}},+ {statement: `CREATE TABLE t2 AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE}},+ {statement: `CREATE VIEW v AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_VIEW}},+ {statement: `CREATE MATERIALIZED VIEW mv AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_VIEW}},+ {statement: `CREATE INDEX ON t (c)`, want: []storepb.StatementType{storepb.StatementType_CREATE_INDEX}},+ {statement: `CREATE SEQUENCE s`, want: []storepb.StatementType{storepb.StatementType_CREATE_SEQUENCE}},+ {statement: `CREATE SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_CREATE_SCHEMA}},+ {statement: `CREATE FUNCTION f() RETURNS INT LANGUAGE SQL AS 'SELECT 1'`, want: []storepb.StatementType{storepb.StatementType_CREATE_FUNCTION}},+ {statement: `CREATE PROCEDURE p() LANGUAGE SQL AS 'SELECT 1'`, want: []storepb.StatementType{storepb.StatementType_CREATE_FUNCTION}},+ {statement: `CREATE TRIGGER tr BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION f()`, want: []storepb.StatementType{storepb.StatementType_CREATE_TRIGGER}},+ {statement: `CREATE EXTENSION IF NOT EXISTS pg_trgm`, want: []storepb.StatementType{storepb.StatementType_CREATE_EXTENSION}},+ {statement: `CREATE TYPE e AS ENUM ('a', 'b')`, want: []storepb.StatementType{storepb.StatementType_CREATE_TYPE}},+ {statement: `DROP DATABASE d`, want: []storepb.StatementType{storepb.StatementType_DROP_DATABASE}},+ {statement: `DROP TABLE t`, want: []storepb.StatementType{storepb.StatementType_DROP_TABLE}},+ {statement: `DROP MATERIALIZED VIEW mv`, want: []storepb.StatementType{storepb.StatementType_DROP_TABLE}},+ {statement: `DROP VIEW v`, want: []storepb.StatementType{storepb.StatementType_DROP_VIEW}},+ {statement: `DROP INDEX t@idx`, want: []storepb.StatementType{storepb.StatementType_DROP_INDEX}},+ {statement: `DROP SEQUENCE s`, want: []storepb.StatementType{storepb.StatementType_DROP_SEQUENCE}},+ {statement: `DROP SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_DROP_SCHEMA}},+ {statement: `DROP TYPE e`, want: []storepb.StatementType{storepb.StatementType_DROP_TYPE}},+ {statement: `DROP TRIGGER tr ON t`, want: []storepb.StatementType{storepb.StatementType_DROP_TRIGGER}},+ {statement: `DROP FUNCTION f`, want: []storepb.StatementType{storepb.StatementType_DROP_FUNCTION}},+ {statement: `TRUNCATE t`, want: []storepb.StatementType{storepb.StatementType_TRUNCATE}},+ {statement: `ALTER DATABASE d RENAME TO d2`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER DATABASE d OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER DATABASE d PRIMARY REGION "us-east1"`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER DATABASE d SURVIVE REGION FAILURE`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER DATABASE d CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER DATABASE d SET timezone = 'UTC'`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},+ {statement: `ALTER ROLE u SET timezone = 'UTC'`},+ {statement: `ALTER TABLE t CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER INDEX t@idx CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_INDEX}},+ {statement: `ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5`},+ {statement: `ALTER TABLE t ADD COLUMN c2 INT`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER TABLE t RENAME COLUMN c TO c2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER TABLE t SET LOCALITY REGIONAL BY ROW`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER TABLE t SET SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER VIEW v OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_VIEW}},+ {statement: `ALTER SEQUENCE s OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_SEQUENCE}},+ {statement: `ALTER INDEX t@idx NOT VISIBLE`, want: []storepb.StatementType{storepb.StatementType_ALTER_INDEX}},+ {statement: `ALTER SEQUENCE s RESTART WITH 100`, want: []storepb.StatementType{storepb.StatementType_ALTER_SEQUENCE}},+ {statement: `ALTER TYPE e ADD VALUE 'c'`, want: []storepb.StatementType{storepb.StatementType_ALTER_TYPE}},+ {statement: `ALTER TABLE t RENAME TO t2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER VIEW v RENAME TO v2`, want: []storepb.StatementType{storepb.StatementType_ALTER_VIEW}},+ {statement: `ALTER MATERIALIZED VIEW mv RENAME TO mv2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},+ {statement: `ALTER SEQUENCE s RENAME TO s2`, want: []storepb.StatementType{storepb.StatementType_RENAME_SEQUENCE}},+ {statement: `ALTER INDEX t@idx RENAME TO idx2`, want: []storepb.StatementType{storepb.StatementType_RENAME_INDEX}},+ {statement: `ALTER SCHEMA app RENAME TO app2`, want: []storepb.StatementType{storepb.StatementType_RENAME_SCHEMA}},+ {statement: `COMMENT ON TABLE t IS 'comment'`, want: []storepb.StatementType{storepb.StatementType_COMMENT}},+ {statement: `COMMENT ON COLUMN t.c IS 'comment'`, want: []storepb.StatementType{storepb.StatementType_COMMENT}},+ {statement: `INSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},+ {statement: `UPSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},+ {statement: `UPDATE t SET c = 1`, want: []storepb.StatementType{storepb.StatementType_UPDATE}},+ {statement: `DELETE FROM t WHERE id = 1`, want: []storepb.StatementType{storepb.StatementType_DELETE}},+ {+ statement: `SET application_name = 'x'; CREATE TABLE t (id INT PRIMARY KEY); SELECT 1; INSERT INTO t VALUES (1)`,+ want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE, storepb.StatementType_INSERT},+ },+ {statement: `WITH d AS (DELETE FROM t RETURNING id) SELECT count(*) FROM d`, want: []storepb.StatementType{storepb.StatementType_DELETE}},+ {statement: `SELECT * FROM [UPDATE t SET c = 1 RETURNING id]`, want: []storepb.StatementType{storepb.StatementType_UPDATE}},+ {+ statement: `WITH d AS (DELETE FROM t RETURNING *) INSERT INTO t2 SELECT * FROM d`,+ want: []storepb.StatementType{storepb.StatementType_INSERT, storepb.StatementType_DELETE},+ },+ {+ statement: `CREATE TABLE t_moved AS SELECT * FROM [DELETE FROM t RETURNING *]`,+ want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE, storepb.StatementType_DELETE},+ },+ {statement: `EXPLAIN ANALYZE DELETE FROM t WHERE id = 1`, want: []storepb.StatementType{storepb.StatementType_DELETE}},+ {statement: `EXPLAIN ANALYZE (VERBOSE) UPSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},+ {statement: `EXPLAIN DELETE FROM t WHERE id = 1`},+ {statement: `SELECT * FROM t`},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ got, err := GetStatementTypes(parseASTs(t, tc.statement))+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ })+ }+}diff --git a/backend/plugin/parser/mysql/explain_plan.go b/backend/plugin/parser/mysql/explain_plan.goindex d35db6059b..9a3e94f5f5 100644--- a/backend/plugin/parser/mysql/explain_plan.go+++ b/backend/plugin/parser/mysql/explain_plan.go@@ -3,105 +3,578 @@ package mysqlimport ("encoding/json""strconv"+ "strings"+ "unicode"++ "github.com/bytebase/omni/mysql/ast"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common")-// GetEstimatedAffectedRowsFromExplainJSON estimates the number of rows affected by an-// UPDATE/DELETE statement from `EXPLAIN FORMAT=JSON` output.-//-// MySQL and MariaDB flag the plan node of each DML target table with "update"/"delete"-// (boolean true on MySQL, integer 1 on MariaDB). Unlike the first "rows" value of the-// tabular EXPLAIN output — which may be the scan estimate of a driving table unrelated-// to the target — the target node carries the optimizer's cardinality estimate for the-// rows actually reaching the modify step. Estimates of multiple target nodes-// (multi-table DELETE/UPDATE) are summed.-//-// The boolean result reports whether any flagged target node with a row estimate was-// found; callers should fall back to the tabular-EXPLAIN heuristic when it is false.-func GetEstimatedAffectedRowsFromExplainJSON(plan string) (int64, bool) {- var root any+// AffectedRowsQuery returns the statement to EXPLAIN for the rows stmt, parsed from statement,+// modifies. MySQL and MariaDB estimate the rows a single-table UPDATE or DELETE scans rather than+// the rows its WHERE clause keeps, so such a statement is explained as a SELECT of those rows.+func AffectedRowsQuery(stmt ast.Node, statement string) string {+ var loc ast.Loc+ var table *ast.TableRef+ // clausesStart is where the WHERE, ORDER BY, and LIMIT clauses begin.+ var clausesStart int+ switch s := stmt.(type) {+ case *ast.UpdateStmt:+ table = singleTableRef(s.Tables)+ if table == nil || len(s.SetList) == 0 {+ return statement+ }+ loc, clausesStart = s.Loc, s.SetList[len(s.SetList)-1].Loc.End+ case *ast.DeleteStmt:+ table = singleTableRef(s.Tables)+ // DELETE writes an alias before PARTITION, which a SELECT writes after it.+ if table == nil || len(s.Using) > 0 || (table.Alias != "" && len(table.Partitions) > 0) {+ return statement+ }+ loc, clausesStart = s.Loc, table.Loc.End+ default:+ return statement+ }+ if loc.Start < 0 || loc.Start > table.Loc.Start || table.Loc.End > clausesStart || clausesStart > loc.End || loc.End > len(statement) {+ return statement+ }+ // omni positions the text after an executable comment as if the comment's markers were removed, so+ // no SELECT is cut from a statement with one. The SELECT would also leave out an optimizer hint+ // before the table, which can change the plan.+ if strings.Contains(statement, "/*!") || strings.Contains(statement, "/*M!") || strings.Contains(statement[loc.Start:table.Loc.Start], "/*+") {+ return statement+ }+ clauses := strings.TrimSpace(statement[clausesStart:loc.End])+ if rest := trimLeadingComments(clauses); rest != "" && !startsWithKeyword(rest, "WHERE", "ORDER", "LIMIT") {+ return statement+ }+ // The text before the statement holds its WITH clause.+ query := statement[:loc.Start] + "SELECT 1 FROM " + statement[table.Loc.Start:table.Loc.End]+ if clauses != "" {+ query += " " + clauses+ }+ return query+}++func singleTableRef(tables []ast.TableExpr) *ast.TableRef {+ if len(tables) != 1 {+ return nil+ }+ if table, ok := tables[0].(*ast.TableRef); ok {+ return table+ }+ return nil+}++// trimLeadingComments removes the whitespace and ordinary comments at the start of text, where -- opens+// a comment only before whitespace or a control character. Executable comments and optimizer hints+// stay, because they can change the statement.+func trimLeadingComments(text string) string {+ for {+ text = strings.TrimLeftFunc(text, unicode.IsSpace)+ switch {+ case strings.HasPrefix(text, "/*!") || strings.HasPrefix(text, "/*M!") || strings.HasPrefix(text, "/*+"):+ return text+ case strings.HasPrefix(text, "/*"):+ end := strings.Index(text[2:], "*/")+ if end < 0 {+ return text+ }+ text = text[2+end+2:]+ case (strings.HasPrefix(text, "--") && (len(text) == 2 || text[2] <= ' ')) || strings.HasPrefix(text, "#"):+ end := strings.IndexByte(text, '\n')+ if end < 0 {+ return ""+ }+ text = text[end+1:]+ default:+ return text+ }+ }+}++func startsWithKeyword(text string, keywords ...string) bool {+ word := text+ if i := strings.IndexFunc(text, func(r rune) bool { return !unicode.IsLetter(r) }); i >= 0 {+ word = text[:i]+ }+ for _, keyword := range keywords {+ if strings.EqualFold(word, keyword) {+ return true+ }+ }+ return false+}++// EstimateAffectedRowsFromExplainJSON returns the planner's estimate of the rows an INSERT,+// REPLACE, UPDATE, or DELETE statement modifies, capped by the statement's LIMIT. The plan is+// MySQL `EXPLAIN FORMAT=JSON` output in explain_json_format_version 1, or MariaDB's, for the+// statement's AffectedRowsQuery.+func EstimateAffectedRowsFromExplainJSON(stmt ast.Node, plan string) (int64, error) {+ var root map[string]anyif err := json.Unmarshal([]byte(plan), &root); err != nil {- return 0, false+ return 0, errors.Wrap(err, "failed to parse EXPLAIN FORMAT=JSON output")}- var total float64+ block, ok := root["query_block"].(map[string]any)+ if !ok {+ return 0, errors.New("unsupported EXPLAIN FORMAT=JSON output: no query_block")+ }++ var rows float64+ var err error+ switch s := stmt.(type) {+ case *ast.InsertStmt:+ // MySQL prints no source plan for a SELECT that reads no table, such as+ // SELECT ... FROM DUAL WHERE NOT EXISTS (...).+ if tableFreeRows, ok := tableFreeSelectRows(s.Select); ok {+ rows = float64(tableFreeRows)+ } else {+ rows, err = queryBlockRows(block)+ }+ case *ast.UpdateStmt:+ rows, err = targetRows(block, updateTargets(s))+ case *ast.DeleteStmt:+ rows, err = targetRows(block, deleteTargets(s))+ default:+ return 0, errors.Errorf("unsupported statement type %T", stmt)+ }+ if err != nil {+ return 0, err+ }+ return CapAffectedRowsByLimit(stmt, rows), nil+}++// CapAffectedRowsByLimit rounds an estimate of the rows an INSERT, REPLACE, UPDATE, or DELETE+// statement modifies and caps it by the statement's LIMIT.+func CapAffectedRowsByLimit(stmt ast.Node, rows float64) int64 {+ var limit *ast.Limit+ switch s := stmt.(type) {+ case *ast.InsertStmt:+ if s.Select != nil {+ limit = s.Select.Limit+ } else if s.TableSource != nil {+ limit = s.TableSource.Limit+ }+ case *ast.UpdateStmt:+ limit = s.Limit+ case *ast.DeleteStmt:+ limit = s.Limit+ default:+ }+ count := common.RoundRows(rows)+ if limitRows, ok := limitCount(limit); ok && limitRows < count {+ return limitRows+ }+ return count+}++// EstimateAffectedRowsFromOceanBaseExplainJSON returns the EST.ROWS estimate from OceanBase+// `EXPLAIN FORMAT=JSON` output: the root operator's, or the largest child operator's when the+// root has no operator.+func EstimateAffectedRowsFromOceanBaseExplainJSON(plan string) (int64, error) {+ var root map[string]any+ if err := json.Unmarshal([]byte(plan), &root); err != nil {+ return 0, errors.Wrap(err, "failed to parse EXPLAIN FORMAT=JSON output")+ }+ if operator, ok := root["OPERATOR"].(string); ok && operator != "" {+ rows, ok := planNumber(root["EST.ROWS"])+ if !ok {+ return 0, errors.Errorf("operator %q has no EST.ROWS", operator)+ }+ return common.RoundRows(rows), nil+ }+ var maxRows float64found := false- var walk func(node any)- walk = func(node any) {+ for key, value := range root {+ child, ok := value.(map[string]any)+ if !ok || !strings.HasPrefix(key, "CHILD_") {+ continue+ }+ operator, ok := child["OPERATOR"].(string)+ if !ok || operator == "" {+ continue+ }+ rows, ok := planNumber(child["EST.ROWS"])+ if !ok {+ return 0, errors.Errorf("operator %q has no EST.ROWS", operator)+ }+ if !found || rows > maxRows {+ maxRows = rows+ found = true+ }+ }+ if !found {+ return 0, errors.New("the plan has no operator")+ }+ return common.RoundRows(maxRows), nil+}++// queryBlockRows returns the planner's estimate of the rows a query block produces. A UNION sums its+// branches, an INTERSECT takes the fewest, and an EXCEPT, or a parenthesized query with its own+// ORDER BY or LIMIT, takes its first.+func queryBlockRows(block map[string]any) (float64, error) {+ for _, operation := range []string{"union_result", "intersect_result", "except_result", "unary_result"} {+ result, ok := block[operation].(map[string]any)+ if !ok {+ continue+ }+ branches, ok := result["query_specifications"].([]any)+ if !ok || len(branches) == 0 {+ return 0, errors.Errorf("%s has no query_specifications", operation)+ }+ var total float64+ for i, branch := range branches {+ specification, ok := branch.(map[string]any)+ if !ok {+ return 0, errors.New("unexpected query_specifications element")+ }+ // MySQL nests a set operation as a branch without a query_block of its own.+ branchBlock, ok := specification["query_block"].(map[string]any)+ if !ok {+ branchBlock = specification+ }+ rows, err := queryBlockRows(branchBlock)+ if err != nil {+ return 0, err+ }+ switch {+ case i == 0 || operation == "union_result":+ total += rows+ case operation == "intersect_result":+ total = min(total, rows)+ default:+ }+ }+ return total, nil+ }+ _, cumulative, err := planJoin(block)+ if err != nil {+ return 0, err+ }+ return cumulative[len(cumulative)-1], nil+}++// dmlTargets holds the names a plan uses for the tables an UPDATE or DELETE modifies.+type dmlTargets struct {+ names []string+ // unqualified counts the multi-table UPDATE assignments to a column without a table qualifier.+ unqualified int+}++// DMLTargetCount returns how many tables stmt can change for each row its join produces: the targets+// of a multi-table UPDATE or DELETE, counting each unqualified assignment as another joined table, up+// to joinedTables, and one for any other statement.+func DMLTargetCount(stmt ast.Node, joinedTables int) int {+ var targets dmlTargets+ switch s := stmt.(type) {+ case *ast.UpdateStmt:+ targets = updateTargets(s)+ case *ast.DeleteStmt:+ targets = deleteTargets(s)+ default:+ return 1+ }+ return max(min(len(targets.names)+targets.unqualified, joinedTables), 1)+}++func updateTargets(stmt *ast.UpdateStmt) dmlTargets {+ if len(stmt.Tables) == 1 {+ if table, ok := stmt.Tables[0].(*ast.TableRef); ok {+ return dmlTargets{names: []string{tableRefPlanName(table)}}+ }+ }+ var targets dmlTargets+ for _, assignment := range stmt.SetList {+ if assignment.Column == nil || assignment.Column.Table == "" {+ targets.unqualified+++ continue+ }+ if !containsFold(targets.names, assignment.Column.Table) {+ targets.names = append(targets.names, assignment.Column.Table)+ }+ }+ return targets+}++func deleteTargets(stmt *ast.DeleteStmt) dmlTargets {+ var targets dmlTargets+ for _, expr := range stmt.Tables {+ table, ok := expr.(*ast.TableRef)+ if !ok {+ continue+ }+ // The multi-table forms list names declared in FROM or USING, so only the single-table+ // form can declare an alias here.+ name := table.Name+ if len(stmt.Using) == 0 {+ name = tableRefPlanName(table)+ }+ targets.names = append(targets.names, name)+ }+ return targets+}++func targetRows(block map[string]any, targets dmlTargets) (float64, error) {+ tables, cumulative, err := planJoin(block)+ if err != nil {+ return 0, err+ }+ final := cumulative[len(cumulative)-1]+ // A plan that produces no rows, such as "Impossible WHERE", prints no target.+ if final == 0 {+ return 0, nil+ }++ // Each target counts min(its cumulative estimate, the join's final estimate): the rows+ // reaching the modification, bounded by the rows read from the target table.+ var total float64+ flagged := false+ for i, table := range tables {+ if planFlag(table["update"]) || planFlag(table["delete"]) {+ total += min(cumulative[i], final)+ flagged = true+ }+ }+ if flagged {+ return total, nil+ }++ // MariaDB flags only single-table targets, and the SELECT a single-table UPDATE or DELETE is+ // explained as flags none, so find the targets by name. A name that no plan table has, such as a+ // view's, or that several have, as when a subquery reads another table of that name, counts the+ // final estimate.+ if len(targets.names) == 0 && targets.unqualified == 0 {+ return 0, errors.New("the plan has no UPDATE or DELETE target")+ }+ for _, name := range targets.names {+ index, matches := -1, 0+ for i, table := range tables {+ if tableName, ok := table["table_name"].(string); ok && strings.EqualFold(tableName, name) {+ index = i+ matches+++ }+ }+ if matches == 1 {+ total += min(cumulative[index], final)+ } else {+ total += final+ }+ }+ // An unqualified column may belong to any joined table, so each counts the final estimate, up to+ // one per joined table.+ total += final * float64(min(targets.unqualified, len(tables)))+ return total, nil+}++// planWrappers are plan members that wrap a query body without changing the tables it joins.+var planWrappers = []string{+ // MySQL.+ "ordering_operation", "grouping_operation", "duplicates_removal", "windowing", "buffer_result",+ // MariaDB.+ "block-nl-join", "read_sorted_file", "filesort", "temporary_table", "window_functions_computation", "buffer",+}++// planJoin returns the tables a query block joins, in join order, with the planner's estimate of+// the rows produced after joining each one. It does not descend into subqueries or derived tables.+func planJoin(block map[string]any) ([]map[string]any, []float64, error) {+ var tables []map[string]any+ var collect func(node any) error+ collect = func(node any) error {switch n := node.(type) {+ case []any:+ for _, element := range n {+ if err := collect(element); err != nil {+ return err+ }+ }+ return nilcase map[string]any:- if isDMLTargetNode(n) {- if estimate, ok := targetNodeRowEstimate(n); ok {- total += estimate- found = true+ // A MySQL INSERT plan puts the target table beside insert_from, which holds the source.+ for _, key := range []string{"insert_from", "nested_loop"} {+ if body, ok := n[key]; ok {+ return collect(body)}}- for _, v := range n {- walk(v)+ if table, ok := n["table"].(map[string]any); ok {+ tables = append(tables, table)+ return nil}- case []any:- for _, v := range n {- walk(v)+ for _, key := range planWrappers {+ if body, ok := n[key]; ok {+ return collect(body)+ }+ }+ if _, ok := n["message"].(string); ok {+ tables = append(tables, n)+ return nil}+ return errors.New("unrecognized EXPLAIN FORMAT=JSON plan node")default:+ return errors.Errorf("unexpected EXPLAIN FORMAT=JSON plan value %v", node)}}- walk(root)- if !found {- return 0, false+ if err := collect(block); err != nil {+ return nil, nil, err+ }+ if len(tables) == 0 {+ return nil, nil, errors.New("the plan has no tables")}- return int64(total), true-}-func isDMLTargetNode(node map[string]any) bool {- for _, key := range []string{"update", "delete"} {- switch v := node[key].(type) {- case bool:- if v {- return true+ cumulative := make([]float64, len(tables))+ previous, fanout := 1.0, 1.0+ for i, table := range tables {+ filtered, ok := planNumber(table["filtered"])+ if !ok {+ filtered = 100+ }+ produced, hasProduced := planNumber(table["rows_produced_per_join"])+ scanned, hasScanned := planNumber(table["rows_examined_per_scan"])+ if !hasScanned {+ scanned, hasScanned = planNumber(table["rows"])+ }+ materialized, isMaterialized := table["materialized_from_subquery"].(map[string]any)+ message, hasMessage := table["message"].(string)+ switch {+ case hasProduced:+ cumulative[i] = produced * fanout+ case hasScanned:+ // MariaDB prints loops; otherwise every row produced so far drives one scan.+ loops, ok := planNumber(table["loops"])+ if !ok {+ loops = previous+ }+ cumulative[i] = loops * scanned * filtered / 100+ case isMaterialized:+ // A MySQL semi-join materialization scan has no row estimate of its own, and the+ // per-join estimates of the tables after it count rows per materialized row.+ subquery, ok := materialized["query_block"].(map[string]any)+ if !ok {+ return nil, nil, errors.New("materialized_from_subquery has no query_block")+ }+ rows, err := queryBlockRows(subquery)+ if err != nil {+ return nil, nil, err}- case float64:- if v != 0 {- return true+ fanout *= rows+ cumulative[i] = previous * rows+ case hasMessage:+ rows, ok := planMessageRows(message)+ if !ok {+ return nil, nil, errors.Errorf("the plan has no row estimate: %s", message)}+ cumulative[i] = previous * rowsdefault:+ if tableName, ok := table["table_name"].(string); ok {+ return nil, nil, errors.Errorf("table %q in the plan has no row estimate", tableName)+ }+ return nil, nil, errors.New("the plan has a table without a row estimate")}+ previous = cumulative[i]}- return false+ return tables, cumulative, nil}-// targetNodeRowEstimate prefers "rows_produced_per_join", the cumulative cardinality-// after all join and filter conditions up to this node. When absent (single-table-// plans, MariaDB), it scales the per-scan row estimate by the node's "filtered"-// percentage.-func targetNodeRowEstimate(node map[string]any) (float64, bool) {- if v, ok := toFloat(node["rows_produced_per_join"]); ok {- return v, true- }- base, ok := toFloat(node["rows_examined_per_scan"])- if !ok {- // MariaDB names the per-scan estimate "rows".- base, ok = toFloat(node["rows"])- }- if !ok {+// planMessageRows returns the rows produced where the plan prints an optimizer message in place+// of a table.+func planMessageRows(message string) (float64, bool) {+ switch message {+ case "Impossible WHERE", "Impossible WHERE noticed after reading const tables",+ "Impossible HAVING", "Impossible HAVING noticed after reading const tables",+ "no matching row in const table", "No matching rows after partition pruning", "Zero limit":+ return 0, true+ case "No tables used", "Select tables optimized away":+ return 1, true+ default:return 0, false}- if filtered, ok := toFloat(node["filtered"]); ok {- base = base * filtered / 100+}++// planFlag reads an update/delete marker, which MySQL prints as true and MariaDB as 1.+func planFlag(value any) bool {+ switch v := value.(type) {+ case bool:+ return v+ case float64:+ return v != 0+ default:+ return false}- return base, true}-func toFloat(v any) (float64, bool) {- switch value := v.(type) {+// planNumber reads a plan number, which MySQL sometimes prints as a string such as "10.00".+func planNumber(value any) (float64, bool) {+ switch v := value.(type) {case float64:- return value, true+ return v, truecase string:- // MySQL renders "filtered" as a string such as "10.00".- f, err := strconv.ParseFloat(value, 64)- if err != nil {+ f, err := strconv.ParseFloat(v, 64)+ return f, err == nil+ default:+ return 0, false+ }+}++// tableFreeSelectRows returns the most rows a SELECT produces when none of its branches reads a+// table: one for each branch, combined as queryBlockRows combines set operations.+func tableFreeSelectRows(sel *ast.SelectStmt) (int, bool) {+ if sel == nil {+ return 0, false+ }+ if sel.ParenSource != nil {+ return tableFreeSelectRows(sel.ParenSource)+ }+ if sel.SetOp != ast.SetOpNone {+ left, leftOK := tableFreeSelectRows(sel.Left)+ right, rightOK := tableFreeSelectRows(sel.Right)+ if !leftOK || !rightOK {return 0, false}- return f, true- default:+ switch sel.SetOp {+ case ast.SetOpIntersect:+ return min(left, right), true+ case ast.SetOpExcept:+ return left, true+ default:+ return left + right, true+ }+ }+ if sel.TableSource != nil || sel.ValuesSource != nil {+ return 0, false+ }+ for _, from := range sel.From {+ table, ok := from.(*ast.TableRef)+ if !ok || table.Schema != "" || !strings.EqualFold(table.Name, "dual") {+ return 0, false+ }+ }+ return 1, true+}++func tableRefPlanName(table *ast.TableRef) string {+ if table.Alias != "" {+ return table.Alias+ }+ return table.Name+}++func containsFold(names []string, name string) bool {+ for _, n := range names {+ if strings.EqualFold(n, name) {+ return true+ }+ }+ return false+}++func limitCount(limit *ast.Limit) (int64, bool) {+ if limit == nil || limit.Count == nil {+ return 0, false+ }+ lit, ok := limit.Count.(*ast.IntLit)+ if !ok || lit.Value < 0 {return 0, false}+ return lit.Value, true}diff --git a/backend/plugin/parser/mysql/explain_plan_test.go b/backend/plugin/parser/mysql/explain_plan_test.goindex 3447b4bffa..191193b880 100644--- a/backend/plugin/parser/mysql/explain_plan_test.go+++ b/backend/plugin/parser/mysql/explain_plan_test.go@@ -1,93 +1,584 @@package mysqlimport (+ "math""os""path/filepath""testing"+ "github.com/bytebase/omni/mysql/ast""github.com/stretchr/testify/require")-// The fixtures are real `EXPLAIN FORMAT=JSON` outputs captured from MySQL 8.0 and-// MariaDB 11 against a 400k-row target_table joined with a 100k-row related_table.-func TestGetEstimatedAffectedRowsFromExplainJSON(t *testing.T) {+// The fixtures are real `EXPLAIN FORMAT=JSON` outputs. Plans over target_table (400k rows) and+// related_table (100k rows) come from MySQL 8.0 and MariaDB 11. The others come from MySQL 8.0.33+// (unprefixed), MySQL 5.7, and MariaDB 11.8 against t (1000 rows), s (100 rows, 10 flagged), big+// (10000 rows, 100 per s row), small (3 rows), and the empty copies t2 and big2.+func TestAffectedRowsQuery(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want string+ }{+ {+ name: "update with filter, order, and limit",+ statement: "UPDATE t SET v = CONCAT(v, 'y'), c = c + 1 WHERE c < 10 ORDER BY id LIMIT 3;",+ want: "SELECT 1 FROM t WHERE c < 10 ORDER BY id LIMIT 3",+ },+ {+ name: "update of an aliased table with an index hint",+ statement: "UPDATE t AS tt FORCE INDEX (idx_c) SET tt.v = 'y' WHERE tt.c < 10",+ want: "SELECT 1 FROM t AS tt FORCE INDEX (idx_c) WHERE tt.c < 10",+ },+ {+ name: "update of every row",+ statement: "UPDATE t SET d = CASE WHEN c > 1 THEN 1 ELSE 2 END;",+ want: "SELECT 1 FROM t",+ },+ {+ name: "update with a comment before WHERE",+ statement: "UPDATE t SET v = (SELECT MAX(v) FROM s) /* latest */ WHERE id = 1",+ want: "SELECT 1 FROM t WHERE id = 1",+ },+ {+ name: "delete with a comment before WHERE",+ statement: "DELETE FROM t -- ticket 42\nWHERE id = 1;",+ want: "SELECT 1 FROM t -- ticket 42\nWHERE id = 1",+ },+ {+ name: "update with a block comment before WHERE",+ statement: "UPDATE t SET c = 1 /* ticket 42 */ WHERE id = 1;",+ want: "SELECT 1 FROM t WHERE id = 1",+ },+ {+ name: "delete with an executable comment",+ statement: "DELETE FROM t /*!80000 WHERE id = 1 */;",+ want: "DELETE FROM t /*!80000 WHERE id = 1 */;",+ },+ {+ name: "update with an optimizer hint",+ statement: "UPDATE /*+ SET_VAR(optimizer_switch='condition_fanout_filter=off') */ t SET c = 1 WHERE id > 5;",+ want: "UPDATE /*+ SET_VAR(optimizer_switch='condition_fanout_filter=off') */ t SET c = 1 WHERE id > 5;",+ },+ {+ name: "delete with an optimizer hint",+ statement: "DELETE /*+ NO_RANGE_OPTIMIZATION(t) */ FROM t WHERE id > 5;",+ want: "DELETE /*+ NO_RANGE_OPTIMIZATION(t) */ FROM t WHERE id > 5;",+ },+ {+ // omni positions the text after an executable comment as if its markers were removed.+ name: "delete with an executable comment in WHERE",+ statement: "DELETE FROM big WHERE /*!50700 v = 0 AND */ s_id < 10000000000;",+ want: "DELETE FROM big WHERE /*!50700 v = 0 AND */ s_id < 10000000000;",+ },+ {+ name: "update with a comment before the table",+ statement: "UPDATE /* ticket 42 */ t SET c = 1 WHERE id > 5;",+ want: "SELECT 1 FROM t WHERE id > 5",+ },+ {+ name: "update with a CTE",+ statement: "WITH x AS (SELECT id FROM s WHERE flag = 1) UPDATE t SET v = 'y' WHERE grp IN (SELECT id FROM x);",+ want: "WITH x AS (SELECT id FROM s WHERE flag = 1) SELECT 1 FROM t WHERE grp IN (SELECT id FROM x)",+ },+ {+ name: "delete from a partition",+ statement: "DELETE LOW_PRIORITY QUICK IGNORE FROM db1.t PARTITION (p0) WHERE c = 1 ORDER BY id DESC LIMIT 5;",+ want: "SELECT 1 FROM db1.t PARTITION (p0) WHERE c = 1 ORDER BY id DESC LIMIT 5",+ },+ {+ name: "delete from an aliased table",+ statement: "DELETE FROM t AS a WHERE a.c = 1;",+ want: "SELECT 1 FROM t AS a WHERE a.c = 1",+ },+ {+ name: "delete of every row",+ statement: "DELETE FROM big;",+ want: "SELECT 1 FROM big",+ },+ {+ name: "delete from a partition of an aliased table",+ statement: "DELETE FROM t AS a PARTITION (p0) WHERE a.c = 1;",+ want: "DELETE FROM t AS a PARTITION (p0) WHERE a.c = 1;",+ },+ {+ name: "multi-table update",+ statement: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1 WHERE s.flag = 1;",+ want: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1 WHERE s.flag = 1;",+ },+ {+ name: "update of a table list",+ statement: "UPDATE t, s SET t.v = 'a' WHERE t.id = s.id;",+ want: "UPDATE t, s SET t.v = 'a' WHERE t.id = s.id;",+ },+ {+ name: "multi-table delete",+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ want: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ },+ {+ name: "delete with USING",+ statement: "DELETE FROM big USING big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ want: "DELETE FROM big USING big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ },+ {+ name: "insert",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ want: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, AffectedRowsQuery(parseSingleStatement(t, tc.statement), tc.statement))+ })+ }+}++func TestEstimateAffectedRowsFromExplainJSON(t *testing.T) {for _, tc := range []struct {fixture string- wantRows int64- wantFound bool+ statement string+ want int64}{{- // UPDATE ... WHERE ... EXISTS rewritten to a semijoin; the target table is- // the last nested_loop node. The first tabular EXPLAIN row would report the- // 100232-row driving-table scan (BYT-9858).+ // The UPDATE target is the last node of a duplicate-weedout semijoin (BYT-9858).fixture: "semijoin_update.json",- wantRows: 1144,- wantFound: true,+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",+ want: 1144,},{- // Single-table DELETE with a range condition: no rows_produced_per_join,- // estimate is rows_examined_per_scan scaled by filtered.+ // Single-table plans have no rows_produced_per_join; the scan estimate is scaled by filtered.fixture: "single_delete.json",- wantRows: 200,- wantFound: true,+ statement: "DELETE FROM target_table WHERE join_key < 50;",+ want: 200,},{- // UPDATE ... ORDER BY ... LIMIT wraps the target node in ordering_operation.fixture: "order_limit_update.json",- wantRows: 200,- wantFound: true,+ statement: "UPDATE target_table SET target_flag = 1 WHERE join_key < 50 ORDER BY id LIMIT 500;",+ want: 200,},{- // NOT EXISTS antijoin: the target table is the first nested_loop node.fixture: "not_exists_update.json",- wantRows: 39964,- wantFound: true,+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND NOT EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",+ want: 39964,},{- // IN (SELECT ...) materialization: the subquery plan lives in- // attached_subqueries and must not contribute to the estimate.+ // The subquery plan in attached_subqueries does not contribute.fixture: "in_materialized_update.json",- wantRows: 399648,- wantFound: true,+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.join_key IN (SELECT t.join_key FROM related_table t WHERE t.filter_column_1 = 'VALUE_A' GROUP BY t.join_key HAVING COUNT(*) > 0);",+ want: 399648,},{- // DELETE o, t FROM ... flags both target nodes; estimates are summed.+ // Both targets are flagged: min(10023, 38163) + min(38163, 38163).fixture: "multi_target_delete.json",- wantRows: 48186,- wantFound: true,+ statement: "DELETE o, t FROM target_table o JOIN related_table t ON t.join_key = o.join_key WHERE t.filter_column_1 = 'VALUE_A';",+ want: 48186,},{fixture: "delete_limit.json",- wantRows: 10,- wantFound: true,+ statement: "DELETE FROM target_table LIMIT 10;",+ want: 10,},{- // MariaDB marks the target with "update": 1 and names the estimate "rows".fixture: "mariadb_update.json",- wantRows: 10000,- wantFound: true,+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",+ want: 10000,+ },+ {+ // The plan scans the flagged target first; the later join filter bounds it: min(11330, 1133).+ fixture: "update_target_first_straight_join.json",+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",+ want: 1133,+ },+ {+ // The derived table's fanout does not raise the target past its own estimate: min(1000, 10000).+ fixture: "update_derived_join.json",+ statement: "UPDATE t JOIN (SELECT grp, COUNT(*) cnt FROM t GROUP BY grp) g ON g.grp = t.grp SET t.v = 'g' WHERE g.cnt > 50;",+ want: 1000,+ },+ {+ // The materialization scan prints no rows, and the target's 113 is per materialized row: 113 * 10.+ fixture: "delete_cte_semijoin_materialization_scan.json",+ statement: "WITH x AS (SELECT id FROM s WHERE flag = 1) DELETE FROM big WHERE s_id IN (SELECT id FROM x);",+ want: 1130,+ },+ {+ // The plans of the SELECT that AffectedRowsQuery explains for a single-table change.+ fixture: "select_for_update_filter.json",+ statement: "UPDATE t SET v = CONCAT(v, 'y') WHERE d = 3;",+ want: 100,+ },+ {+ fixture: "mariadb_select_for_delete_range_filter.json",+ statement: "DELETE FROM t WHERE c < 10 AND d = 3;",+ want: 14,+ },+ {+ fixture: "mariadb_select_for_delete_all_rows.json",+ statement: "DELETE FROM big;",+ want: 10000,},{- // SELECT plans have no update/delete node; callers must fall back.+ // A target the plan does not name, such as a view, counts the final estimate.fixture: "select_only.json",- wantRows: 0,- wantFound: false,+ statement: "UPDATE other_table SET target_flag = 1;",+ want: 200,+ },+ {+ // UNION ALL of 3 joined rows and the first EXCEPT branch's 500.+ fixture: "insert_select_union_except.json",+ statement: "INSERT INTO t2 SELECT t.* FROM small JOIN t USING (id) UNION ALL (SELECT * FROM t WHERE c < 50 EXCEPT SELECT * FROM t WHERE d = 1);",+ want: 503,+ },+ {+ // UNION ALL of 3 joined rows and the smaller INTERSECT branch's 100.+ fixture: "insert_select_union_intersect.json",+ statement: "INSERT INTO t2 SELECT t.* FROM small JOIN t USING (id) UNION ALL (SELECT * FROM t WHERE c < 50 INTERSECT SELECT * FROM t WHERE d = 1);",+ want: 103,+ },+ {+ fixture: "insert_select_parenthesized_limit.json",+ statement: "INSERT INTO t2 (SELECT * FROM t ORDER BY c LIMIT 500) ORDER BY id LIMIT 400;",+ want: 400,+ },+ {+ fixture: "update_impossible_where.json",+ statement: "UPDATE t SET v = 'y' WHERE 1 = 0;",+ want: 0,+ },+ {+ // MySQL 5.7 prints the row estimate beside the "Deleting all rows" message.+ fixture: "mysql57_delete_all_rows.json",+ statement: "DELETE FROM big;",+ want: 10195,+ },+ {+ fixture: "insert_select_filter.json",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ want: 100,+ },+ {+ fixture: "insert_select_join.json",+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",+ want: 1133,+ },+ {+ // Only the first branch sits under insert_from: 100 + 899.+ fixture: "insert_select_union_all.json",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE c < 10 UNION ALL SELECT * FROM t WHERE c >= 10;",+ want: 999,+ },+ {+ fixture: "replace_select_join.json",+ statement: "REPLACE INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",+ want: 1133,+ },+ {+ fixture: "insert_table.json",+ statement: "INSERT INTO t2 TABLE t;",+ want: 1000,+ },+ {+ fixture: "insert_select_join.json",+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1 LIMIT 30;",+ want: 30,+ },+ {+ // The lookup into the materialized subquery has no per-join estimate.+ fixture: "insert_select_semijoin_materialization_lookup.json",+ statement: "INSERT INTO t2 WITH x AS (SELECT id FROM s WHERE flag = 1) SELECT * FROM t WHERE grp IN (SELECT id FROM x);",+ want: 1000,+ },+ {+ fixture: "insert_select_buffer_result.json",+ statement: "INSERT INTO t2 SELECT t.* FROM t LEFT JOIN t2 ON t2.id = t.id WHERE t2.id IS NULL;",+ want: 1000,+ },+ {+ fixture: "insert_select_impossible_where.json",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE 1 = 0;",+ want: 0,+ },+ {+ // MariaDB marks no multi-table target, so the target is found by name.+ fixture: "mariadb_delete_join.json",+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",+ want: 1000,+ },+ {+ fixture: "mariadb_delete_semijoin.json",+ statement: "DELETE FROM big WHERE s_id IN (SELECT id FROM s WHERE flag = 1);",+ want: 1000,+ },+ {+ // The lookup into the materialized subquery prints no loops.+ fixture: "mariadb_delete_semijoin_materialization.json",+ statement: "DELETE FROM s WHERE id IN (SELECT s_id FROM big WHERE v = 0);",+ want: 100,+ },+ {+ // min(10000, 1000) for the target big.+ fixture: "mariadb_update_straight_join.json",+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",+ want: 1000,+ },+ {+ // big: min(1000, 1000); s: min(10, 1000).+ fixture: "mariadb_update_two_targets.json",+ statement: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1, s.flag = 1 WHERE s.flag = 1;",+ want: 1010,+ },+ {+ fixture: "mariadb_update_join_unqualified_column.json",+ statement: "UPDATE big AS b JOIN s AS x ON b.s_id = x.id SET v = 1 WHERE x.flag = 1;",+ want: 1000,+ },+ {+ // Either joined table may own each unqualified column: 2 * 1000.+ fixture: "mariadb_update_join_unqualified_column.json",+ statement: "UPDATE big AS b JOIN s AS x ON b.s_id = x.id SET v = 1, flag = 0 WHERE x.flag = 1;",+ want: 2000,+ },+ {+ fixture: "mariadb_delete_impossible_where.json",+ statement: "DELETE FROM t WHERE 1 = 0;",+ want: 0,+ },+ {+ // 1000 rows filtered to 14.3%.+ fixture: "mariadb_insert_select_filter.json",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",+ want: 143,+ },+ {+ fixture: "mariadb_insert_select_join.json",+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",+ want: 1000,+ },+ {+ fixture: "mariadb_insert_select_union_all.json",+ statement: "INSERT INTO t2 SELECT * FROM t WHERE c < 10 UNION ALL SELECT * FROM t WHERE c >= 10;",+ want: 1000,+ },+ {+ fixture: "mariadb_insert_select_block_nl_join.json",+ statement: "INSERT INTO t2 SELECT t.* FROM t JOIN s ON t.d = s.flag WHERE s.id < 50;",+ want: 49000,+ },+ {+ fixture: "mariadb_insert_select_order_limit.json",+ statement: "INSERT INTO t2 SELECT * FROM t ORDER BY d LIMIT 50;",+ want: 50,+ },+ {+ fixture: "mariadb_insert_select_window.json",+ statement: "INSERT INTO t2 (id, c, d, grp, v) SELECT id, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY id), d, grp, v FROM t WHERE c < 10;",+ want: 100,+ },+ {+ fixture: "mariadb_insert_select_no_tables.json",+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a';",+ want: 1,+ },+ {+ // MySQL prints only the INSERT target for a source that reads no table.+ fixture: "insert_select_from_dual_not_exists.json",+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a' FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE id = 1);",+ want: 1,+ },+ {+ fixture: "insert_select_union_without_tables.json",+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a' UNION ALL SELECT 2, 2, 2, 2, 'b';",+ want: 2,},} {t.Run(tc.fixture, func(t *testing.T) {- plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", tc.fixture))+ got, err := EstimateAffectedRowsFromExplainJSON(parseSingleStatement(t, tc.statement), readExplainPlanFixture(t, tc.fixture))require.NoError(t, err)- rows, found := GetEstimatedAffectedRowsFromExplainJSON(string(plan))- require.Equal(t, tc.wantFound, found)- require.Equal(t, tc.wantRows, rows)+ require.Equal(t, tc.want, got)+ })+ }+}++func TestTableFreeSelectRows(t *testing.T) {+ for _, tc := range []struct {+ statement string+ want int+ wantOK bool+ }{+ {statement: "INSERT INTO t SELECT 1 UNION ALL SELECT 2 UNION SELECT 3", want: 3, wantOK: true},+ {statement: "INSERT INTO t SELECT 1 INTERSECT SELECT 1 FROM DUAL", want: 1, wantOK: true},+ {statement: "INSERT INTO t SELECT 1 EXCEPT SELECT 2", want: 1, wantOK: true},+ {statement: "INSERT INTO t SELECT 1 UNION ALL SELECT id FROM s"},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ insert, ok := parseSingleStatement(t, tc.statement).(*ast.InsertStmt)+ require.True(t, ok)+ got, ok := tableFreeSelectRows(insert.Select)+ require.Equal(t, tc.wantOK, ok)+ require.Equal(t, tc.want, got)+ })+ }+}++func TestDMLTargetCount(t *testing.T) {+ for _, tc := range []struct {+ statement string+ want int+ }{+ {statement: "UPDATE t SET a = 1, b = 2 WHERE id = 1", want: 1},+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.x = 1, b.y = 1", want: 2},+ {statement: "UPDATE a JOIN b ON a.id = b.id SET x = 1, y = 2, z = 3", want: 2},+ {statement: "DELETE a, b FROM a JOIN b ON a.id = b.id", want: 2},+ {statement: "DELETE FROM t WHERE id = 1", want: 1},+ {statement: "INSERT INTO t SELECT * FROM s", want: 1},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ require.Equal(t, tc.want, DMLTargetCount(parseSingleStatement(t, tc.statement), 2))+ })+ }+}++func TestCapAffectedRowsByLimit(t *testing.T) {+ for _, tc := range []struct {+ statement string+ rows float64+ want int64+ }{+ {statement: "INSERT INTO t2 SELECT t.id FROM t, big, big AS big3", rows: 1e30, want: math.MaxInt64},+ {statement: "INSERT INTO t2 SELECT t.id FROM t, big, big AS big3 LIMIT 10", rows: 1e30, want: 10},+ {statement: "DELETE FROM t WHERE id > 0 LIMIT 5", rows: 2.5, want: 3},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ require.Equal(t, tc.want, CapAffectedRowsByLimit(parseSingleStatement(t, tc.statement), tc.rows))})}}-func TestGetEstimatedAffectedRowsFromExplainJSONInvalidInput(t *testing.T) {- for _, plan := range []string{"", "not json", "[]", `{"query_block":{}}`} {- rows, found := GetEstimatedAffectedRowsFromExplainJSON(plan)- require.False(t, found, plan)- require.Zero(t, rows, plan)+func TestEstimateAffectedRowsFromExplainJSONError(t *testing.T) {+ for _, tc := range []struct {+ name string+ plan string+ statement string+ wantErr string+ }{+ {+ // MySQL 8.0 omits insert_from when the SELECT computes window functions.+ name: "insert without a source plan",+ plan: readExplainPlanFixture(t, "insert_select_window_no_source.json"),+ statement: "INSERT INTO t2 (id, c, d, grp, v) SELECT id, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY id), d, grp, v FROM t WHERE c < 10;",+ wantErr: `table "t2" in the plan has no row estimate`,+ },+ {+ name: "MariaDB delete of all rows",+ plan: readExplainPlanFixture(t, "mariadb_delete_all_rows.json"),+ statement: "DELETE FROM big;",+ wantErr: "the plan has no row estimate: Deleting all rows",+ },+ {+ name: "format version 2",+ plan: `{"query": "/* select#1 */ delete from t", "operation": "Delete from t", "access_type": "table", "estimated_rows": 1000}`,+ statement: "DELETE FROM t;",+ wantErr: "no query_block",+ },+ {+ name: "unrecognized plan node",+ plan: `{"query_block": {"select_id": 1, "future_operation": {"table": {"delete": true, "table_name": "t", "rows_examined_per_scan": 10}}}}`,+ statement: "DELETE FROM t;",+ wantErr: "unrecognized EXPLAIN FORMAT=JSON plan node",+ },+ {+ name: "not JSON",+ plan: "-> Delete from t",+ statement: "DELETE FROM t;",+ wantErr: "failed to parse EXPLAIN FORMAT=JSON output",+ },+ {+ name: "unsupported statement",+ plan: readExplainPlanFixture(t, "select_only.json"),+ statement: "SELECT * FROM target_table;",+ wantErr: "unsupported statement type",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ _, err := EstimateAffectedRowsFromExplainJSON(parseSingleStatement(t, tc.statement), tc.plan)+ require.ErrorContains(t, err, tc.wantErr)+ })+ }+}++func TestEstimateAffectedRowsFromOceanBaseExplainJSON(t *testing.T) {+ for _, tc := range []struct {+ name string+ plan string+ want int64+ }{+ {+ name: "root operator",+ plan: `{"ID":0,"OPERATOR":"UPDATE","NAME":"","EST.ROWS":1000,"EST.TIME(us)":7680,"output":"","CHILD_1":{"ID":1,"OPERATOR":"TABLE RANGE SCAN","NAME":"dba_test_1","EST.ROWS":1200,"EST.TIME(us)":91}}`,+ want: 1000,+ },+ {+ name: "fractional estimate",+ plan: `{"ID":0,"OPERATOR":"DELETE","NAME":"","EST.ROWS":2.6,"EST.TIME(us)":20}`,+ want: 3,+ },+ {+ name: "largest child operator without a root operator",+ plan: `{"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t1","EST.ROWS":40},"CHILD_2":{"ID":2,"OPERATOR":"TABLE FULL SCAN","NAME":"t2","EST.ROWS":70}}`,+ want: 70,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, err := EstimateAffectedRowsFromOceanBaseExplainJSON(tc.plan)+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ })+ }++ for _, tc := range []struct {+ name string+ plan string+ wantErr string+ }{+ {+ name: "root operator without an estimate",+ plan: `{"ID":0,"OPERATOR":"UPDATE","NAME":""}`,+ wantErr: `operator "UPDATE" has no EST.ROWS`,+ },+ {+ name: "child operator without an estimate",+ plan: `{"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t1"}}`,+ wantErr: `operator "TABLE FULL SCAN" has no EST.ROWS`,+ },+ {+ name: "no operator",+ plan: `{}`,+ wantErr: "the plan has no operator",+ },+ {+ name: "empty output",+ plan: "",+ wantErr: "failed to parse EXPLAIN FORMAT=JSON output",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ _, err := EstimateAffectedRowsFromOceanBaseExplainJSON(tc.plan)+ require.ErrorContains(t, err, tc.wantErr)+ })}}++func readExplainPlanFixture(t *testing.T, fixture string) string {+ t.Helper()+ plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", fixture))+ require.NoError(t, err)+ return string(plan)+}++func parseSingleStatement(t *testing.T, statement string) ast.Node {+ t.Helper()+ list, err := ParseMySQL(statement)+ require.NoError(t, err)+ require.Len(t, list.Items, 1)+ return list.Items[0]+}diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.goindex 940b6eee3a..9c1ae0df7b 100644--- a/backend/plugin/parser/mysql/resource_change.go+++ b/backend/plugin/parser/mysql/resource_change.go@@ -6,7 +6,6 @@ import ("github.com/bytebase/omni/mysql/ast""github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -24,7 +23,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *modelchangedResources := model.NewChangedResources(dbMetadata)var dmlCount, insertCount int- var sampleDMLs []string+ var dmlStatements []string// addObjectDatabase records a database-only write target for a non-table object DDL// (view/function/routine/trigger) ONLY when the object's own name carries an explicit@@ -109,19 +108,20 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *modelinsertCount += len(n.Values)continue}- dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))+ // INSERT ... SET inserts exactly one row, and EXPLAIN gives no estimate for it.+ if len(n.SetList) > 0 {+ insertCount+++ continue}+ dmlCount+++ dmlStatements = append(dmlStatements, omniStatementText(omniAST))case *ast.UpdateStmt:for _, resource := range extractTableExprs(n.Tables, currentDatabase) {changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)}dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))- }+ dmlStatements = append(dmlStatements, omniStatementText(omniAST))case *ast.DeleteStmt:for _, resource := range extractTableExprs(n.Tables, currentDatabase) {@@ -131,9 +131,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *modelchangedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)}dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))- }+ dmlStatements = append(dmlStatements, omniStatementText(omniAST))// Non-table object DDL: gate by the object's own explicit database qualifier only.// CREATE TRIGGER is intentionally skipped — its AST name is unqualified (a bare string);@@ -161,7 +159,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *modelreturn &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: sampleDMLs,+ DMLStatements: dmlStatements,DMLCount: dmlCount,InsertCount: insertCount,}, nildiff --git a/backend/plugin/parser/mysql/resource_change_test.go b/backend/plugin/parser/mysql/resource_change_test.goindex a983e521a1..a083913bb2 100644--- a/backend/plugin/parser/mysql/resource_change_test.go+++ b/backend/plugin/parser/mysql/resource_change_test.go@@ -16,6 +16,7 @@ func TestExtractChangedResources(t *testing.T) {ALTER TABLE t1 ADD COLUMN c1 INT;RENAME TABLE t1 TO t2;INSERT INTO t1 (c1) VALUES (1), (5);+ INSERT INTO t1 SET c1 = 7;UPDATE t1 SET c1 = 5;CREATE PROCEDURE getUser(id INT) SELECT * FROM hello WHERE uid = id;`@@ -38,11 +39,11 @@ func TestExtractChangedResources(t *testing.T) {)want := &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: []string{+ DMLStatements: []string{"UPDATE t1 SET c1 = 5;",},DMLCount: 1,- InsertCount: 2,+ InsertCount: 3,}stmts, err := base.ParseStatements(storepb.Engine_MYSQL, statement)diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.json b/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.jsonnew file mode 100644index 0000000000..68d2c466da--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"122.70"},"nested_loop":[{"table":{"table_name":"<subquery2>","access_type":"ALL","materialized_from_subquery":{"using_temporary_table":true,"query_block":{"table":{"table_name":"x","access_type":"ALL","rows_examined_per_scan":10,"rows_produced_per_join":10,"filtered":"100.00","cost_info":{"read_cost":"2.62","eval_cost":"1.00","prefix_cost":"3.62","data_read_per_join":"160"},"used_columns":["id"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"10.25"},"table":{"table_name":"s","access_type":"ALL","rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}}}}}}},{"table":{"delete":true,"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["<subquery2>.id"],"rows_examined_per_scan":113,"rows_produced_per_join":113,"filtered":"100.00","cost_info":{"read_cost":"2.77","eval_cost":"11.33","prefix_cost":"122.70","data_read_per_join":"1K"},"used_columns":["id","s_id","v"]}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.jsonnew file mode 100644index 0000000000..7c9f98664b--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"451.75"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"},"insert_from":{"buffer_result":{"using_temporary_table":true,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"t2","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.t.id"],"rows_examined_per_scan":1,"rows_produced_per_join":1000,"filtered":"100.00","not_exists":true,"using_index":true,"cost_info":{"read_cost":"250.00","eval_cost":"100.00","prefix_cost":"451.75","data_read_per_join":"101K"},"used_columns":["id"],"attached_condition":"<if>(found_match(t2), (`probe`.`t2`.`id` is null), true)"}}]}}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.jsonnew file mode 100644index 0000000000..3c0c012135--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"101.75"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.75","eval_cost":"10.00","prefix_cost":"101.75","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d` = 3)"}}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.jsonnew file mode 100644index 0000000000..c61d121fa7--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"table":{"insert":true,"table_name":"t2","access_type":"ALL","attached_subqueries":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"message":"no matching row in const table"}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.jsonnew file mode 100644index 0000000000..fa8e537378--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"message":"Impossible WHERE"}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.jsonnew file mode 100644index 0000000000..0c9c2cda0a--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"406.80"},"table":{"insert":true,"table_name":"big2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["probe.s.id"],"rows_examined_per_scan":113,"rows_produced_per_join":1133,"filtered":"100.00","cost_info":{"read_cost":"283.25","eval_cost":"113.30","prefix_cost":"406.80","data_read_per_join":"17K"},"used_columns":["id","s_id","v"]}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.jsonnew file mode 100644index 0000000000..0678ec504d--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.json@@ -0,0 +1 @@+{"query_block":{"unary_result":{"using_temporary_table":true,"select_id":2,"table_name":"<ordered/limited1>","access_type":"ALL","using_filesort":true,"query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"101.25"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"},"insert_from":{"ordering_operation":{"using_filesort":true,"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.25","eval_cost":"100.00","prefix_cost":"101.25","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}}}}}]}}}\ No newline at end of filediff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.jsonnew file mode 100644index 0000000000..3aaf15e7e0--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"207.37"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"<subquery2>","access_type":"eq_ref","key":"<auto_distinct_key>","key_length":"4","ref":["probe.t.grp"],"rows_examined_per_scan":1,"materialized_from_subquery":{"using_temporary_table":true,"query_block":{"table":{"table_name":"x","access_type":"ALL","rows_examined_per_scan":10,"rows_produced_per_join":10,"filtered":"100.00","cost_info":{"read_cost":"2.62","eval_cost":"1.00","prefix_cost":"3.62","data_read_per_join":"160"},"used_columns":["id"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"10.25"},"table":{"table_name":"s","access_type":"ALL","rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}}}}}}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.jsonnew file mode 100644index 0000000000..e724d9f532--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.json@@ -0,0 +1 @@+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":3,"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"45.26"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","used_key_parts":["c"],"key_length":"4","rows_examined_per_scan":100,"rows_produced_per_join":100,"filtered":"100.00","index_condition":"(`probe`.`t`.`c` < 10)","cost_info":{"read_cost":"35.26","eval_cost":"10.00","prefix_cost":"45.26","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"]}}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.75"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":899,"filtered":"90.00","cost_info":{"read_cost":"11.75","eval_cost":"90.00","prefix_cost":"101.75","data_read_per_join":"91K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c` >= 10)"}}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.jsonnew file mode 100644index 0000000000..ed37ba9574--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.json@@ -0,0 +1 @@+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":5,"table_name":"<union1,4>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"1.60"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"small","access_type":"index","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","rows_examined_per_scan":3,"rows_produced_per_join":3,"filtered":"100.00","using_index":true,"cost_info":{"read_cost":"0.25","eval_cost":"0.30","prefix_cost":"0.55","data_read_per_join":"48"},"used_columns":["id"]}},{"table":{"table_name":"t","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.small.id"],"rows_examined_per_scan":1,"rows_produced_per_join":3,"filtered":"100.00","cost_info":{"read_cost":"0.75","eval_cost":"0.30","prefix_cost":"1.60","data_read_per_join":"312"},"used_columns":["id","c","d","grp","v"]}}]}}},{"except_result":{"using_temporary_table":true,"select_id":4,"table_name":"<except2,3>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":500,"filtered":"50.00","cost_info":{"read_cost":"51.25","eval_cost":"50.00","prefix_cost":"101.25","data_read_per_join":"50K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c`<50)"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.25","eval_cost":"10.00","prefix_cost":"101.25","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d`=1)"}}}]}}]}}}\ No newline at end of filediff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.jsonnew file mode 100644index 0000000000..9a9372c62a--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.json@@ -0,0 +1 @@+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":5,"table_name":"<union1,4>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"1.60"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"small","access_type":"index","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","rows_examined_per_scan":3,"rows_produced_per_join":3,"filtered":"100.00","using_index":true,"cost_info":{"read_cost":"0.25","eval_cost":"0.30","prefix_cost":"0.55","data_read_per_join":"48"},"used_columns":["id"]}},{"table":{"table_name":"t","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.small.id"],"rows_examined_per_scan":1,"rows_produced_per_join":3,"filtered":"100.00","cost_info":{"read_cost":"0.75","eval_cost":"0.30","prefix_cost":"1.60","data_read_per_join":"312"},"used_columns":["id","c","d","grp","v"]}}]}}},{"intersect_result":{"using_temporary_table":true,"select_id":4,"table_name":"<intersect2,3>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":500,"filtered":"50.00","cost_info":{"read_cost":"51.25","eval_cost":"50.00","prefix_cost":"101.25","data_read_per_join":"50K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c`<50)"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.25","eval_cost":"10.00","prefix_cost":"101.25","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d`=1)"}}}]}}]}}}\ No newline at end of filediff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.jsonnew file mode 100644index 0000000000..2d13676ced--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.json@@ -0,0 +1 @@+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":3,"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"table":{"insert":true,"table_name":"t2","access_type":"ALL"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"message":"No tables used"}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.jsonnew file mode 100644index 0000000000..20a033bf09--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"145.26"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.jsonnew file mode 100644index 0000000000..021181aa2c--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"101.75"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.jsonnew file mode 100644index 0000000000..1a38f4cc72--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"table":{"message":"Deleting all rows"}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.jsonnew file mode 100644index 0000000000..d1b86a015c--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"table":{"message":"Impossible WHERE"}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.jsonnew file mode 100644index 0000000000..b2717d8279--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":0.1878414,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":0.1606866,"filtered":100}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.jsonnew file mode 100644index 0000000000..b2717d8279--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":0.1878414,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":0.1606866,"filtered":100}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.jsonnew file mode 100644index 0000000000..e22ea5820f--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":3.3635308,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":100}},{"table":{"table_name":"<subquery2>","access_type":"eq_ref","possible_keys":["distinct_key"],"key":"distinct_key","key_length":"4","used_key_parts":["s_id"],"ref":["func"],"rows":1,"filtered":100,"materialized":{"unique":1,"query_block":{"select_id":2,"nested_loop":[{"table":{"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"loops":1,"rows":10000,"cost":1.6588056,"filtered":100,"attached_condition":"big.v = 0"}}]}}}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.jsonnew file mode 100644index 0000000000..dec3d3d35a--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":4.741674304,"nested_loop":[{"table":{"table_name":"s","access_type":"range","possible_keys":["PRIMARY"],"key":"PRIMARY","key_length":"4","used_key_parts":["id"],"loops":1,"rows":49,"cost":0.01183867,"filtered":100,"attached_condition":"s.`id` < 50"}},{"block-nl-join":{"table":{"table_name":"t","access_type":"ALL","loops":49,"rows":1000,"cost":4.729835634,"filtered":100},"buffer_type":"flat","buffer_size":"348","join_type":"BNL","attached_condition":"t.d = s.flag"}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.jsonnew file mode 100644index 0000000000..9d04f7c04d--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":0.1764192,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","loops":1,"rows":1000,"cost":0.1764192,"filtered":14.30000019,"attached_condition":"t.d = 3"}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.jsonnew file mode 100644index 0000000000..d729014ba7--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.jsonnew file mode 100644index 0000000000..6d7280a4e0--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"table":{"message":"No tables used"}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.jsonnew file mode 100644index 0000000000..c2b88282d3--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":0.1764192,"nested_loop":[{"read_sorted_file":{"filesort":{"sort_key":"t.d","table":{"table_name":"t","access_type":"ALL","loops":1,"rows":1000,"cost":0.1764192,"filtered":100}}}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.jsonnew file mode 100644index 0000000000..36bcf5c677--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.json@@ -0,0 +1 @@+{"query_block":{"union_result":{"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"query_block":{"select_id":1,"cost":0.10505784,"nested_loop":[{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","key_length":"4","used_key_parts":["c"],"loops":1,"rows":100,"cost":0.10505784,"filtered":100,"index_condition":"t.c < 10"}}]}},{"query_block":{"select_id":2,"operation":"UNION","cost":0.1764192,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"loops":1,"rows":1000,"cost":0.1764192,"filtered":90,"attached_condition":"t.c >= 10"}}]}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.jsonnew file mode 100644index 0000000000..987f70c55c--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":0.10505784,"window_functions_computation":{"sorts":[{"filesort":{"sort_key":"t.grp, t.`id`"}}],"temporary_table":{"nested_loop":[{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","key_length":"4","used_key_parts":["c"],"loops":1,"rows":100,"cost":0.10505784,"filtered":100,"index_condition":"t.c < 10"}}]}}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.jsonnew file mode 100644index 0000000000..cc51e45541--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.json@@ -0,0 +1,22 @@+{+ "query_block": {+ "select_id": 1,+ "cost": 1.47862472,+ "nested_loop": [+ {+ "table": {+ "table_name": "big",+ "access_type": "index",+ "key": "idx_s",+ "key_length": "4",+ "used_key_parts": ["s_id"],+ "loops": 1,+ "rows": 10000,+ "cost": 1.47862472,+ "filtered": 100,+ "using_index": true+ }+ }+ ]+ }+}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.jsonnew file mode 100644index 0000000000..aa661e48bf--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.json@@ -0,0 +1,24 @@+{+ "query_block": {+ "select_id": 1,+ "cost": 0.10505784,+ "nested_loop": [+ {+ "table": {+ "table_name": "t",+ "access_type": "range",+ "possible_keys": ["idx_c"],+ "key": "idx_c",+ "key_length": "4",+ "used_key_parts": ["c"],+ "loops": 1,+ "rows": 100,+ "cost": 0.10505784,+ "filtered": 14.30000019,+ "index_condition": "t.c < 10",+ "attached_condition": "t.d = 3"+ }+ }+ ]+ }+}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.jsonnew file mode 100644index 0000000000..c54018b34f--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"x","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"x.flag = 1"}},{"table":{"table_name":"b","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.x.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.jsonnew file mode 100644index 0000000000..d36b956fe1--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":10.6040248,"nested_loop":[{"table":{"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"loops":1,"rows":10000,"cost":1.6588056,"filtered":100}},{"table":{"table_name":"s","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","key_length":"4","used_key_parts":["id"],"ref":["probe.big.s_id"],"loops":10000,"rows":1,"cost":8.9452192,"filtered":10,"attached_condition":"s.flag = 1"}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.jsonnew file mode 100644index 0000000000..d729014ba7--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.jsonnew file mode 100644index 0000000000..250ed37d3c--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"table":{"delete":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":10195,"filtered":"100.00","message":"Deleting all rows"}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.jsonnew file mode 100644index 0000000000..560946e571--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"406.80"},"table":{"replace":true,"table_name":"big2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["probe.s.id"],"rows_examined_per_scan":113,"rows_produced_per_join":1133,"filtered":"100.00","cost_info":{"read_cost":"283.25","eval_cost":"113.30","prefix_cost":"406.80","data_read_per_join":"17K"},"used_columns":["id","s_id","v"]}}]}}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.jsonnew file mode 100644index 0000000000..400efad569--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.json@@ -0,0 +1,25 @@+{+ "query_block": {+ "select_id": 1,+ "cost_info": {+ "query_cost": "101.75"+ },+ "table": {+ "table_name": "t",+ "access_type": "ALL",+ "rows_examined_per_scan": 1000,+ "rows_produced_per_join": 100,+ "filtered": "10.00",+ "cost_info": {+ "read_cost": "91.75",+ "eval_cost": "10.00",+ "prefix_cost": "101.75",+ "data_read_per_join": "10K"+ },+ "used_columns": [+ "d"+ ],+ "attached_condition": "(`probe`.`t`.`d` = 3)"+ }+ }+}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.jsonnew file mode 100644index 0000000000..855666c036--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"3601.75"},"nested_loop":[{"table":{"update":true,"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"g","access_type":"ref","possible_keys":["<auto_key0>"],"key":"<auto_key0>","used_key_parts":["grp"],"key_length":"4","ref":["probe.t.grp"],"rows_examined_per_scan":10,"rows_produced_per_join":10000,"filtered":"100.00","cost_info":{"read_cost":"2500.00","eval_cost":"1000.00","prefix_cost":"3601.75","data_read_per_join":"234K"},"used_columns":["grp","cnt"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.75"},"grouping_operation":{"using_temporary_table":true,"using_filesort":false,"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","grp"]}}}}}}]}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.jsonnew file mode 100644index 0000000000..fa8e537378--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"message":"Impossible WHERE"}}diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.jsonnew file mode 100644index 0000000000..5d6aff7284--- /dev/null+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.json@@ -0,0 +1 @@+{"query_block":{"select_id":1,"cost_info":{"query_cost":"5122.75"},"nested_loop":[{"table":{"update":true,"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00","cost_info":{"read_cost":"24.25","eval_cost":"1133.00","prefix_cost":"1157.25","data_read_per_join":"177K"},"used_columns":["id","s_id","v"]}},{"table":{"table_name":"s","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.big.s_id"],"rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00","cost_info":{"read_cost":"2832.50","eval_cost":"113.30","prefix_cost":"5122.75","data_read_per_join":"8K"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}]}}diff --git a/backend/plugin/parser/pg/explain_plan.go b/backend/plugin/parser/pg/explain_plan.gonew file mode 100644index 0000000000..4a4ee8fbae--- /dev/null+++ b/backend/plugin/parser/pg/explain_plan.go@@ -0,0 +1,112 @@+package pg++import (+ "encoding/json"++ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+)++type explainPlanNode struct {+ NodeType string `json:"Node Type"`+ ParentRelationship string `json:"Parent Relationship"`+ PlanRows *float64 `json:"Plan Rows"`+ Plans []explainPlanNode `json:"Plans"`+}++// GetEstimatedAffectedRowsFromExplainJSON returns the planner's estimate of the rows modified+// by the INSERT, UPDATE, DELETE, and MERGE operations in `EXPLAIN (FORMAT JSON)` output. It+// sums every ModifyTable node, including those of data-modifying CTEs, and returns an error+// when the plan has none or one cannot be interpreted.+func GetEstimatedAffectedRowsFromExplainJSON(plan string) (int64, error) {+ roots, err := parseExplainJSON(plan)+ if err != nil {+ return 0, err+ }++ var total float64+ modifyTableCount := 0+ var walk func(node *explainPlanNode) error+ walk = func(node *explainPlanNode) error {+ if node.NodeType == "ModifyTable" {+ rows, err := getModifiedRows(node)+ if err != nil {+ return err+ }+ total += rows+ modifyTableCount+++ }+ for i := range node.Plans {+ if err := walk(&node.Plans[i]); err != nil {+ return err+ }+ }+ return nil+ }+ for _, root := range roots {+ if err := walk(root); err != nil {+ return 0, err+ }+ }+ if modifyTableCount == 0 {+ return 0, errors.New("the plan has no ModifyTable node")+ }+ return common.RoundRows(total), nil+}++// GetEstimatedInsertedRowsFromExplainJSON returns the planner's estimate of the rows a top-level+// INSERT adds in `EXPLAIN (FORMAT JSON)` output, without the rows its data-modifying CTEs change.+func GetEstimatedInsertedRowsFromExplainJSON(plan string) (int64, error) {+ roots, err := parseExplainJSON(plan)+ if err != nil {+ return 0, err+ }+ if len(roots) != 1 || roots[0].NodeType != "ModifyTable" {+ return 0, errors.New("the plan root is not a ModifyTable node")+ }+ rows, err := getModifiedRows(roots[0])+ if err != nil {+ return 0, err+ }+ return common.RoundRows(rows), nil+}++// parseExplainJSON returns the root plan node of each statement in `EXPLAIN (FORMAT JSON)` output.+func parseExplainJSON(plan string) ([]*explainPlanNode, error) {+ var statements []struct {+ Plan *explainPlanNode `json:"Plan"`+ }+ if err := json.Unmarshal([]byte(plan), &statements); err != nil {+ return nil, errors.Wrap(err, "failed to parse the EXPLAIN (FORMAT JSON) output")+ }+ var roots []*explainPlanNode+ for _, statement := range statements {+ if statement.Plan != nil {+ roots = append(roots, statement.Plan)+ }+ }+ return roots, nil+}++// getModifiedRows reads the estimate from the subplans feeding a ModifyTable node. PostgreSQL 14+// and later label its only subplan "Outer"; earlier versions label one "Member" per target table.+func getModifiedRows(modifyTable *explainPlanNode) (float64, error) {+ var rows float64+ found := false+ for _, child := range modifyTable.Plans {+ switch child.ParentRelationship {+ case "Outer", "Member":+ if child.PlanRows == nil {+ return 0, errors.Errorf("the %s node feeding ModifyTable has no Plan Rows", child.NodeType)+ }+ rows += *child.PlanRows+ found = true+ default:+ }+ }+ if !found {+ return 0, errors.New("the ModifyTable node has no subplan feeding it")+ }+ return rows, nil+}diff --git a/backend/plugin/parser/pg/explain_plan_test.go b/backend/plugin/parser/pg/explain_plan_test.gonew file mode 100644index 0000000000..6438d58e42--- /dev/null+++ b/backend/plugin/parser/pg/explain_plan_test.go@@ -0,0 +1,100 @@+package pg++import (+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"+)++// The fixtures are real `EXPLAIN (FORMAT JSON)` outputs captured from PostgreSQL 17 after ANALYZE:+// t has 1000 rows, s has 100 rows (10 flagged), big has 100 rows per s id, and parted and the+// inheritance parent inh_parent each hold 1000 rows.+func TestGetEstimatedAffectedRowsFromExplainJSON(t *testing.T) {+ for _, tc := range []struct {+ fixture string+ wantRows int64+ }{+ {fixture: "update.json", wantRows: 100},+ {fixture: "delete_using_join.json", wantRows: 1000},+ {fixture: "update_from_join.json", wantRows: 1000},+ {fixture: "delete_in_subquery.json", wantRows: 1000},+ {fixture: "update_correlated_set_subquery.json", wantRows: 100},+ {fixture: "update_initplan.json", wantRows: 3333},+ {fixture: "update_partitioned.json", wantRows: 100},+ {fixture: "delete_inheritance_parent.json", wantRows: 101},+ {fixture: "insert_select.json", wantRows: 100},+ {fixture: "insert_select_on_conflict_do_nothing.json", wantRows: 1000},+ {fixture: "insert_values_on_conflict.json", wantRows: 2},+ {fixture: "merge.json", wantRows: 10000},+ {fixture: "cte_delete_select.json", wantRows: 5000},+ // The CTE deletes 5000 rows and the outer INSERT inserts them again.+ {fixture: "cte_delete_insert.json", wantRows: 10000},+ } {+ t.Run(tc.fixture, func(t *testing.T) {+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(readExplainPlanFixture(t, tc.fixture))+ require.NoError(t, err)+ require.Equal(t, tc.wantRows, rows)+ })+ }+}++func TestGetEstimatedInsertedRowsFromExplainJSON(t *testing.T) {+ for _, tc := range []struct {+ fixture string+ wantRows int64+ }{+ {fixture: "insert_select.json", wantRows: 100},+ // The outer INSERT adds the 5000 rows its CTE deletes.+ {fixture: "cte_delete_insert.json", wantRows: 5000},+ } {+ t.Run(tc.fixture, func(t *testing.T) {+ rows, err := GetEstimatedInsertedRowsFromExplainJSON(readExplainPlanFixture(t, tc.fixture))+ require.NoError(t, err)+ require.Equal(t, tc.wantRows, rows)+ })+ }++ _, err := GetEstimatedInsertedRowsFromExplainJSON(readExplainPlanFixture(t, "select.json"))+ require.EqualError(t, err, "the plan root is not a ModifyTable node")+}++func TestGetEstimatedAffectedRowsFromExplainJSONMemberSubplans(t *testing.T) {+ // PostgreSQL 13 and earlier plan a partitioned UPDATE as one ModifyTable subplan per partition.+ const plan = `[{"Plan": {"Node Type": "ModifyTable", "Operation": "Update", "Plan Rows": 0, "Plans": [+ {"Node Type": "Seq Scan", "Parent Relationship": "Member", "Relation Name": "parted_p1", "Plan Rows": 50},+ {"Node Type": "Seq Scan", "Parent Relationship": "Member", "Relation Name": "parted_p2", "Plan Rows": 50}+ ]}}]`+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(plan)+ require.NoError(t, err)+ require.Equal(t, int64(100), rows)+}++func TestGetEstimatedAffectedRowsFromExplainJSONError(t *testing.T) {+ for name, plan := range map[string]string{+ "select": readExplainPlanFixture(t, "select.json"),+ "costs off": readExplainPlanFixture(t, "update_costs_off.json"),+ "empty": "",+ "not json": "not json",+ "no statements": "[]",+ "text format": "Update on t (cost=5.05..13.30 rows=0 width=0)",+ "modify table without subplan": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0}}]`,+ "modify table with only an init plan": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [+ {"Node Type": "Result", "Parent Relationship": "InitPlan", "Plan Rows": 1}+ ]}}]`,+ } {+ t.Run(name, func(t *testing.T) {+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(plan)+ require.Error(t, err)+ require.Zero(t, rows)+ })+ }+}++func readExplainPlanFixture(t *testing.T, name string) string {+ t.Helper()+ plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", name))+ require.NoError(t, err)+ return string(plan)+}diff --git a/backend/plugin/parser/pg/query_antlr.go b/backend/plugin/parser/pg/query_antlr.goindex 31f855d74c..1b7e248006 100644--- a/backend/plugin/parser/pg/query_antlr.go+++ b/backend/plugin/parser/pg/query_antlr.go@@ -1,8 +1,6 @@package pgimport (- "strings"-"github.com/bytebase/omni/pg/ast""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -43,7 +41,7 @@ func validateQueryANTLR(statement string) (bool, bool, error) {}case *ast.ExplainStmt:- if isExplainAnalyze(n) {+ if isExplainAnalyzeOmni(n) {// EXPLAIN ANALYZE executes the query, so it must be a read-only SELECT.sel, ok := n.Query.(*ast.SelectStmt)if !ok || isWriteSelect(sel) {@@ -66,28 +64,12 @@ func validateQueryANTLR(statement string) (bool, bool, error) {return true, !hasExecute, nil}-// isExplainAnalyze checks if an ExplainStmt has the ANALYZE option.-func isExplainAnalyze(n *ast.ExplainStmt) bool {- if n.Options == nil {- return false- }- for _, item := range n.Options.Items {- if de, ok := item.(*ast.DefElem); ok {- if strings.EqualFold(de.Defname, "analyze") {- return true- }- }- }- return false-}-// isWriteSelect reports whether a SelectStmt actually writes — and so must not take// the read-only editor path. A SELECT writes if it has an INTO target (SELECT ...// INTO creates a table) or any of its CTE terms is data-modifying.//-// This is the unified write-detection primitive for the read-only gate;-// classifyQueryType (query_type.go) keeps reporting the root statement type for its-// own consumers — only the detection is shared, not the classifiers' outputs.+// The read-only gate uses it; classifyQueryType (query_type.go) shares containsWriteCTE+// so the permission check treats the same SELECTs as DML.func isWriteSelect(n *ast.SelectStmt) bool {if n == nil {return falsediff --git a/backend/plugin/parser/pg/query_test.go b/backend/plugin/parser/pg/query_test.goindex fecc57bda5..30872749b3 100644--- a/backend/plugin/parser/pg/query_test.go+++ b/backend/plugin/parser/pg/query_test.go@@ -38,6 +38,16 @@ func TestValidateSQLForEditor(t *testing.T) {valid: false,allQuery: false,},+ {+ sql: `EXPLAIN (ANALYZE off) DELETE FROM t`,+ valid: true,+ allQuery: true,+ },+ {+ sql: `EXPLAIN (ANALYZE on) DELETE FROM t`,+ valid: false,+ allQuery: false,+ },{sql: `With t as (diff --git a/backend/plugin/parser/pg/query_type.go b/backend/plugin/parser/pg/query_type.goindex 5983d96211..ea3821b32f 100644--- a/backend/plugin/parser/pg/query_type.go+++ b/backend/plugin/parser/pg/query_type.go@@ -25,6 +25,10 @@ func classifyQueryType(node ast.Node, allSystems bool) (queryType base.QueryTypeif hasOmniIntoClause(n) {return base.DDL, false}+ // A data-modifying CTE writes, so the SELECT needs DML permission.+ if containsWriteCTE(n) {+ return base.DML, false+ }if allSystems {return base.SelectInfoSchema, false}@@ -120,19 +124,47 @@ func omniIntoClause(n *ast.SelectStmt) *ast.IntoClause {return omniIntoClause(n.Rarg)}-// isExplainAnalyzeOmni checks if an ExplainStmt has the ANALYZE option.+// isExplainAnalyzeOmni reports whether an ExplainStmt executes its query: its last ANALYZE option+// is not FALSE, OFF, or 0, the values PostgreSQL reads as false.func isExplainAnalyzeOmni(n *ast.ExplainStmt) bool {if n.Options == nil {return false}+ analyze := falsefor _, item := range n.Options.Items {- if de, ok := item.(*ast.DefElem); ok {- if strings.EqualFold(de.Defname, "analyze") {- return true- }+ de, ok := item.(*ast.DefElem)+ if !ok || !strings.EqualFold(de.Defname, "analyze") {+ continue+ }+ switch arg := de.Arg.(type) {+ case *ast.String:+ analyze = !strings.EqualFold(arg.Str, "false") && !strings.EqualFold(arg.Str, "off")+ case *ast.Integer:+ analyze = arg.Ival != 0+ default:+ analyze = true}}- return false+ return analyze+}++// UnwrapExplainAnalyze returns the statement that an EXPLAIN ANALYZE executes and that statement's text+// within text, the text of the EXPLAIN. It returns any other node and text unchanged.+func UnwrapExplainAnalyze(node ast.Node, text string) (ast.Node, string) {+ explain, ok := node.(*ast.ExplainStmt)+ if !ok || !isExplainAnalyzeOmni(explain) {+ return node, text+ }+ // The statement runs from its WITH clause, which the location of a SELECT leaves out, to the end+ // of the EXPLAIN, which its location can also leave out, as for ORDER BY.+ start := ast.NodeLoc(explain.Query).Start+ if with := getWithClause(explain.Query); with != nil && with.Loc.Start >= 0 && with.Loc.Start < start {+ start = with.Loc.Start+ }+ if start < 0 || start >= explain.Loc.End || explain.Loc.End > len(text) {+ return explain.Query, text+ }+ return explain.Query, text[start:explain.Loc.End]}// classifyExplainedQuery returns the QueryType for the query inside EXPLAIN ANALYZE.@@ -145,8 +177,11 @@ func classifyExplainedQuery(query ast.Node) base.QueryType {if hasOmniIntoClause(n) {return base.DDL}+ if containsWriteCTE(n) {+ return base.DML+ }return base.Select- case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:return base.DMLcase *ast.DeclareCursorStmt:return base.Selectdiff --git a/backend/plugin/parser/pg/resource_change.go b/backend/plugin/parser/pg/resource_change.goindex ba5c699c1e..c711498867 100644--- a/backend/plugin/parser/pg/resource_change.go+++ b/backend/plugin/parser/pg/resource_change.go@@ -1,12 +1,12 @@package pgimport (+ "slices""strings""github.com/bytebase/omni/pg/ast""github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -14,7 +14,6 @@ import (func init() {base.RegisterExtractChangedResourcesFunc(storepb.Engine_POSTGRES, extractChangedResources)- base.RegisterExtractChangedResourcesFunc(storepb.Engine_COCKROACHDB, extractChangedResources)}func extractChangedResources(database string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {@@ -31,13 +30,42 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *return &base.ChangeSummary{ChangedResources: changedResources,DMLCount: 0,- SampleDMLS: []string{},+ DMLStatements: []string{},InsertCount: 0,}, nil}var dmlCount, insertCount int- var sampleDMLs []string+ var dmlStatements []string+ initialSearchPath := searchPath+ // sessionSearchPath leaves out SET LOCAL, which lasts until the transaction ends, and+ // transactionSearchPath is the session search path that ROLLBACK restores.+ sessionSearchPath, transactionSearchPath := searchPath, searchPath+ inTransaction := false+ sampleDML := func(text string) {+ dmlCount+++ text = strings.TrimSpace(text)+ if !strings.HasSuffix(text, ";") {+ text += ";"+ }+ // The EXPLAIN connection starts with the initial search path, so a changed one is replayed.+ if !slices.Equal(searchPath, initialSearchPath) {+ text = base.WithSearchPath(text, searchPath)+ }+ dmlStatements = append(dmlStatements, text)+ }+ addTarget := func(rv *ast.RangeVar, affectedTable bool) {+ db, schema, table := extractExistingRangeVarNames(rv, database, searchPath, dbMetadata)+ changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, affectedTable)+ }+ // addDataModifyingCTETargets reports whether the WITH clause modifies data.+ addDataModifyingCTETargets := func(with *ast.WithClause) bool {+ targets := getDataModifyingCTETargets(with)+ for _, rv := range targets {+ addTarget(rv, false)+ }+ return len(targets) > 0+ }for _, unifiedAST := range asts {omniAST, ok := unifiedAST.(*OmniAST)@@ -47,11 +75,39 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *if omniAST.Node == nil {continue}+ node, text := UnwrapExplainAnalyze(omniAST.Node, omniAST.Text)++ switch n := node.(type) {+ case *ast.TransactionStmt:+ switch n.Kind {+ case ast.TRANS_STMT_BEGIN, ast.TRANS_STMT_START:+ // A BEGIN inside a transaction only warns.+ if !inTransaction {+ transactionSearchPath, inTransaction = sessionSearchPath, true+ }+ case ast.TRANS_STMT_COMMIT, ast.TRANS_STMT_ROLLBACK:+ // A ROLLBACK outside a transaction only warns.+ if n.Kind == ast.TRANS_STMT_ROLLBACK && inTransaction {+ sessionSearchPath = transactionSearchPath+ }+ // COMMIT AND CHAIN and ROLLBACK AND CHAIN start the next transaction from here.+ searchPath, transactionSearchPath, inTransaction = sessionSearchPath, sessionSearchPath, n.Chain+ default:+ }++ case *ast.DiscardStmt:+ if n.Target == ast.DISCARD_ALL {+ searchPath, sessionSearchPath = initialSearchPath, initialSearchPath+ }- switch n := omniAST.Node.(type) {case *ast.VariableSetStmt:- if strings.EqualFold(n.Name, "search_path") && n.Args != nil {- var newSearchPath []string+ var newSearchPath []string+ if n.Kind == ast.VAR_RESET_ALL || (strings.EqualFold(n.Name, "search_path") && (n.Kind == ast.VAR_RESET || n.Kind == ast.VAR_SET_DEFAULT)) {+ newSearchPath = initialSearchPath+ } else if strings.EqualFold(n.Name, "search_path") && n.Kind == ast.VAR_SET_CURRENT {+ // SET ... FROM CURRENT makes the current search path, which a SET LOCAL may have set, the session's.+ newSearchPath = searchPath+ } else if strings.EqualFold(n.Name, "search_path") && n.Args != nil {for _, arg := range n.Args.Items {if ac, ok := arg.(*ast.A_Const); ok {if s, ok := ac.Val.(*ast.String); ok {@@ -59,8 +115,11 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *}}}- if len(newSearchPath) > 0 {- searchPath = newSearchPath+ }+ if len(newSearchPath) > 0 {+ searchPath = newSearchPath+ if !n.IsLocal {+ sessionSearchPath = newSearchPath}}@@ -90,12 +149,24 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *}case *ast.RenameStmt:- if n.Relation != nil {+ if n.Relation == nil {+ continue+ }+ // Newname names a table only when the table itself is renamed.+ switch n.RenameType {+ case ast.OBJECT_TABLE, ast.OBJECT_MATVIEW:db, schema, oldTableName := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: oldTableName}, true)if n.Newname != "" {changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: n.Newname}, false)}+ case ast.OBJECT_COLUMN, ast.OBJECT_TABCONSTRAINT:+ if n.RelationType == ast.OBJECT_TABLE || n.RelationType == ast.OBJECT_MATVIEW {+ addTarget(n.Relation, true)+ }+ case ast.OBJECT_TRIGGER, ast.OBJECT_RULE, ast.OBJECT_POLICY:+ addTarget(n.Relation, true)+ default:}case *ast.IndexStmt:@@ -105,34 +176,32 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *}case *ast.InsertStmt:+ hasDataModifyingCTE := addDataModifyingCTETargets(n.WithClause)if n.Relation != nil {- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)+ addTarget(n.Relation, false)}- // Count insert rows from VALUES- if sel, ok := n.SelectStmt.(*ast.SelectStmt); ok && sel.ValuesLists != nil {- insertCount += len(sel.ValuesLists.Items)+ // A data-modifying CTE makes the whole statement a sample, whose estimate already+ // includes the VALUES rows.+ if rows, ok := getInsertValuesRowCount(n); ok && !hasDataModifyingCTE {+ insertCount += rows+ } else {+ sampleDML(text)}case *ast.UpdateStmt:+ addDataModifyingCTETargets(n.WithClause)if n.Relation != nil {- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)- }- dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, getOmniStatementText(omniAST))+ addTarget(n.Relation, false)}+ sampleDML(text)case *ast.DeleteStmt:+ addDataModifyingCTETargets(n.WithClause)if n.Relation != nil {- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)- }- dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, getOmniStatementText(omniAST))+ addTarget(n.Relation, false)}+ sampleDML(text)+case *ast.TruncateStmt:if n.Relations != nil {for _, item := range n.Relations.Items {@@ -146,16 +215,20 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *}case *ast.MergeStmt:+ addDataModifyingCTETargets(n.WithClause)if n.Relation != nil {- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)+ addTarget(n.Relation, false)}+ sampleDML(text)case *ast.CreateTableAsStmt:if n.Into != nil && n.Into.Rel != nil {db, schema, table := extractNewRangeVarNames(n.Into.Rel, database, searchPath, dbMetadata)changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)}+ if query, ok := n.Query.(*ast.SelectStmt); ok && addDataModifyingCTETargets(query.WithClause) {+ sampleDML(text)+ }case *ast.SelectStmt:// SELECT ... INTO target_table is a write (classified DDL via the INTO clause,@@ -164,6 +237,9 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *db, schema, table := extractNewRangeVarNames(into.Rel, database, searchPath, dbMetadata)changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)}+ if addDataModifyingCTETargets(n.WithClause) {+ sampleDML(text)+ }default:}@@ -172,11 +248,76 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *return &base.ChangeSummary{ChangedResources: changedResources,DMLCount: dmlCount,- SampleDMLS: sampleDMLs,+ DMLStatements: dmlStatements,InsertCount: insertCount,}, nil}+// getInsertValuesRowCount returns the rows inserted by INSERT ... VALUES or INSERT ... DEFAULT+// VALUES; ok is false when a query supplies the rows.+func getInsertValuesRowCount(n *ast.InsertStmt) (rows int, ok bool) {+ if n.SelectStmt == nil {+ return 1, true+ }+ if sel, isSelect := n.SelectStmt.(*ast.SelectStmt); isSelect && sel.ValuesLists != nil {+ return len(sel.ValuesLists.Items), true+ }+ return 0, false+}++// getWithClause returns the WITH clause of a statement, the only one where PostgreSQL allows+// data-modifying statements.+func getWithClause(node ast.Node) *ast.WithClause {+ switch n := node.(type) {+ case *ast.SelectStmt:+ return n.WithClause+ case *ast.InsertStmt:+ return n.WithClause+ case *ast.UpdateStmt:+ return n.WithClause+ case *ast.DeleteStmt:+ return n.WithClause+ case *ast.MergeStmt:+ return n.WithClause+ case *ast.CreateTableAsStmt:+ return getWithClause(n.Query)+ default:+ return nil+ }+}++// getDataModifyingCTETargets returns the tables that INSERT, UPDATE, DELETE, and MERGE statements in+// a WITH clause modify. PostgreSQL only allows data-modifying statements in a top-level WITH.+func getDataModifyingCTETargets(with *ast.WithClause) []*ast.RangeVar {+ if with == nil || with.Ctes == nil {+ return nil+ }+ var targets []*ast.RangeVar+ for _, item := range with.Ctes.Items {+ cte, ok := item.(*ast.CommonTableExpr)+ if !ok {+ continue+ }+ var target *ast.RangeVar+ switch query := cte.Ctequery.(type) {+ case *ast.InsertStmt:+ target = query.Relation+ case *ast.UpdateStmt:+ target = query.Relation+ case *ast.DeleteStmt:+ target = query.Relation+ case *ast.MergeStmt:+ target = query.Relation+ default:+ continue+ }+ if target != nil {+ targets = append(targets, target)+ }+ }+ return targets+}+// extractRangeVarNames extracts database, schema, table from a RangeVar with defaults.func extractRangeVarNames(rv *ast.RangeVar, defaultDB string, searchPath []string) (string, string, string) {db := rv.Catalogname@@ -300,12 +441,3 @@ func extractNameListParts(nameList *ast.List, defaultDB string) (string, string,return defaultDB, "", ""}}--// getOmniStatementText returns the text of a statement from OmniAST, including semicolon.-func getOmniStatementText(omniAST *OmniAST) string {- text := strings.TrimSpace(omniAST.Text)- if !strings.HasSuffix(text, ";") {- text += ";"- }- return text-}diff --git a/backend/plugin/parser/pg/resource_change_test.go b/backend/plugin/parser/pg/resource_change_test.goindex 172175b4ef..4605b95c38 100644--- a/backend/plugin/parser/pg/resource_change_test.go+++ b/backend/plugin/parser/pg/resource_change_test.go@@ -51,7 +51,7 @@ func TestExtractChangedResources(t *testing.T) {want := &base.ChangeSummary{ChangedResources: changedResources,DMLCount: 1,- SampleDMLS: []string{"UPDATE t1 SET c1 = 5;"},+ DMLStatements: []string{"UPDATE t1 SET c1 = 5;"},InsertCount: 2,}@@ -63,6 +63,76 @@ func TestExtractChangedResources(t *testing.T) {require.Equal(t, want, got)}+func TestExtractChangedResourcesReplaysSearchPath(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ SearchPath: `"$user", public`,+ Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}, {Name: "app"}},+ }, nil, nil, storepb.Engine_POSTGRES, true /* caseSensitive */)+ const statement = `UPDATE t SET c = 1;+SET search_path TO app;+UPDATE t SET c = 2;+SET search_path TO public;+UPDATE t SET c = 3;+SET search_path TO app;+RESET search_path;+UPDATE t SET c = 4;+SET search_path TO app;+SET search_path TO DEFAULT;+UPDATE t SET c = 5;+BEGIN;+SET LOCAL search_path TO app;+UPDATE t SET c = 6;+COMMIT;+UPDATE t SET c = 7;+BEGIN;+SET search_path TO app;+ROLLBACK;+UPDATE t SET c = 8;+BEGIN;+SET search_path TO app;+COMMIT AND CHAIN;+ROLLBACK;+UPDATE t SET c = 9;+DISCARD ALL;+UPDATE t SET c = 10;+SET search_path TO app;+BEGIN;+SET search_path TO other;+BEGIN;+SET search_path TO third;+ROLLBACK;+UPDATE t SET c = 11;+SET search_path TO other;+SET LOCAL search_path TO app;+ROLLBACK;+UPDATE t SET c = 12;+BEGIN;+SET LOCAL search_path TO app;+SET search_path FROM CURRENT;+COMMIT;+UPDATE t SET c = 13;`+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)+ require.NoError(t, err)+ got, err := extractChangedResources("db", "", dbMetadata, base.ExtractASTs(stmts), statement)+ require.NoError(t, err)+ require.Equal(t, []string{+ "UPDATE t SET c = 1;",+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 2;",+ "UPDATE t SET c = 3;",+ "UPDATE t SET c = 4;",+ "UPDATE t SET c = 5;",+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 6;",+ "UPDATE t SET c = 7;",+ "UPDATE t SET c = 8;",+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 9;",+ "UPDATE t SET c = 10;",+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 11;",+ "SET LOCAL search_path TO \"other\";\nUPDATE t SET c = 12;",+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 13;",+ }, got.DMLStatements)+}+func TestExtractChangedResourcesSelectedSchemaFallsBackToPublicForExistingTarget(t *testing.T) {const statement = `INSERT INTO customer VALUES (1);`@@ -159,6 +229,182 @@ func TestExtractChangedResourcesMerge(t *testing.T) {require.Equal(t, want, got.ChangedResources)}+func TestExtractChangedResourcesDMLCounts(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)+ for _, tc := range []struct {+ name string+ statement string+ wantDMLCount int+ wantInsertCount int+ wantTables []string+ }{+ {+ name: "insert values",+ statement: `INSERT INTO t VALUES (1), (2), (3);`,+ wantInsertCount: 3,+ wantTables: []string{"t"},+ },+ {+ name: "insert values on conflict",+ statement: `INSERT INTO t VALUES (1), (2) ON CONFLICT (id) DO UPDATE SET c = 1;`,+ wantInsertCount: 2,+ wantTables: []string{"t"},+ },+ {+ name: "insert default values",+ statement: `INSERT INTO t DEFAULT VALUES;`,+ wantInsertCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "insert select",+ statement: `INSERT INTO t SELECT * FROM t2 ON CONFLICT DO NOTHING;`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "insert select with read-only CTE",+ statement: `WITH x AS (SELECT id FROM t2) INSERT INTO t SELECT id FROM x;`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "merge",+ statement: `MERGE INTO t USING t2 ON t.id = t2.id WHEN MATCHED THEN UPDATE SET c = t2.c;`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "data-modifying CTE feeding insert",+ statement: `WITH d AS (DELETE FROM t WHERE id > 0 RETURNING *) INSERT INTO t2 SELECT * FROM d;`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "data-modifying CTE feeding insert values",+ statement: `WITH i AS (INSERT INTO t VALUES (1) RETURNING id) INSERT INTO t2 VALUES (2), (3);`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2"},+ },+ {+ name: "data-modifying CTE in select",+ statement: `WITH d AS (DELETE FROM t RETURNING 1) SELECT count(*) FROM d;`,+ wantDMLCount: 1,+ wantTables: []string{"t"},+ },+ {+ name: "data-modifying CTEs feeding delete",+ statement: `WITH u AS (UPDATE t SET c = 1 RETURNING id), m AS (MERGE INTO t3 USING t2 ON t3.id = t2.id WHEN MATCHED THEN DELETE) DELETE FROM t2 WHERE id IN (SELECT id FROM u);`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t2", "t3"},+ },+ {+ name: "data-modifying CTE in create table as",+ statement: `CREATE TABLE t_moved AS WITH d AS (DELETE FROM t RETURNING *) SELECT * FROM d;`,+ wantDMLCount: 1,+ wantTables: []string{"t", "t_moved"},+ },+ {+ name: "read-only CTE in select",+ statement: `WITH x AS (SELECT id FROM t) SELECT * FROM x;`,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ want := model.NewChangedResources(dbMetadata)+ for _, table := range tc.wantTables {+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)+ }+ var wantSamples []string+ if tc.wantDMLCount > 0 {+ wantSamples = []string{tc.statement}+ }++ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, tc.statement)+ require.NoError(t, err)+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)+ require.NoError(t, err)+ require.Equal(t, &base.ChangeSummary{+ ChangedResources: want,+ DMLCount: tc.wantDMLCount,+ DMLStatements: wantSamples,+ InsertCount: tc.wantInsertCount,+ }, got)+ })+ }+}++func TestExtractChangedResourcesExplainAnalyze(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)+ const statement = `EXPLAIN (ANALYZE, BUFFERS) WITH d AS (DELETE FROM t RETURNING id) INSERT INTO t2 SELECT id FROM d;+EXPLAIN ANALYZE MERGE INTO t3 USING t ON t3.id = t.id WHEN MATCHED THEN DELETE;+EXPLAIN UPDATE t4 SET c = 1;+EXPLAIN (ANALYZE FALSE) DELETE FROM t5;+EXPLAIN ANALYZE WITH d AS (DELETE FROM t6 RETURNING id) SELECT count(*) FROM d ORDER BY 1;`++ want := model.NewChangedResources(dbMetadata)+ for _, table := range []string{"t", "t2", "t3", "t6"} {+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)+ }+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)+ require.NoError(t, err)+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), statement)+ require.NoError(t, err)+ require.Equal(t, &base.ChangeSummary{+ ChangedResources: want,+ DMLCount: 3,+ DMLStatements: []string{+ "WITH d AS (DELETE FROM t RETURNING id) INSERT INTO t2 SELECT id FROM d;",+ "MERGE INTO t3 USING t ON t3.id = t.id WHEN MATCHED THEN DELETE;",+ "WITH d AS (DELETE FROM t6 RETURNING id) SELECT count(*) FROM d ORDER BY 1;",+ },+ }, got)+}++func TestExtractChangedResourcesRename(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {Name: "t", RowCount: 10},+ {Name: "t2", RowCount: 1000},+ {Name: "t3", RowCount: 1_000_000},+ },+ }},+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)+ for _, tc := range []struct {+ statement string+ wantTables []string+ wantAffectedRows int64+ }{+ {statement: `ALTER TABLE t RENAME TO t_new;`, wantTables: []string{"t", "t_new"}, wantAffectedRows: 10},+ {statement: `ALTER TABLE t2 RENAME COLUMN c TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},+ {statement: `ALTER TABLE t2 RENAME c TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},+ {statement: `ALTER TABLE t2 RENAME CONSTRAINT t2_pkey TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},+ {statement: `ALTER TRIGGER trg ON t2 RENAME TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},+ {statement: `ALTER INDEX t2_idx RENAME TO t3;`},+ {statement: `ALTER VIEW v RENAME COLUMN c TO t3;`},+ } {+ t.Run(tc.statement, func(t *testing.T) {+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, tc.statement)+ require.NoError(t, err)+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)+ require.NoError(t, err)++ var gotTables []string+ for _, database := range got.ChangedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ gotTables = append(gotTables, table.GetName())+ }+ }+ }+ require.Equal(t, tc.wantTables, gotTables)+ require.Equal(t, tc.wantAffectedRows, got.ChangedResources.CountAffectedTableRows())+ })+ }+}+func TestExtractChangedResourcesCTAS(t *testing.T) {dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)for _, tc := range []struct {diff --git a/backend/plugin/parser/pg/statement_type.go b/backend/plugin/parser/pg/statement_type.goindex 148870e55e..85957bbf10 100644--- a/backend/plugin/parser/pg/statement_type.go+++ b/backend/plugin/parser/pg/statement_type.go@@ -34,8 +34,8 @@ func GetStatementTypes(asts []base.AST) ([]StatementTypeWithPosition, error) {continue}- stmtType := classifyStatementType(omniAST.Node)- if stmtType == storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED {+ stmtTypes := classifyStatementTypes(omniAST.Node)+ if len(stmtTypes) == 0 {continue}@@ -50,11 +50,13 @@ func GetStatementTypes(asts []base.AST) ([]StatementTypeWithPosition, error) {endLine += strings.Count(omniAST.Text, "\n")}- allResults = append(allResults, StatementTypeWithPosition{- Type: stmtType,- Line: endLine,- Text: omniAST.Text,- })+ for _, stmtType := range stmtTypes {+ allResults = append(allResults, StatementTypeWithPosition{+ Type: stmtType,+ Line: endLine,+ Text: omniAST.Text,+ })+ }}return allResults, nildiff --git a/backend/plugin/parser/pg/statement_type_antlr.go b/backend/plugin/parser/pg/statement_type_antlr.goindex 0430f12df9..1a50b7349d 100644--- a/backend/plugin/parser/pg/statement_type_antlr.go+++ b/backend/plugin/parser/pg/statement_type_antlr.go@@ -1,11 +1,34 @@package pgimport (+ "slices"+"github.com/bytebase/omni/pg/ast"storepb "github.com/bytebase/bytebase/backend/generated-go/store")+// classifyStatementTypes returns the type of the statement followed by the types of its+// data-modifying CTEs, each type once.+func classifyStatementTypes(node ast.Node) []storepb.StatementType {+ node, _ = UnwrapExplainAnalyze(node, "")+ var types []storepb.StatementType+ add := func(statementType storepb.StatementType) {+ if statementType != storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED && !slices.Contains(types, statementType) {+ types = append(types, statementType)+ }+ }+ add(classifyStatementType(node))+ if with := getWithClause(node); with != nil && with.Ctes != nil {+ for _, item := range with.Ctes.Items {+ if cte, ok := item.(*ast.CommonTableExpr); ok {+ add(classifyStatementType(cte.Ctequery))+ }+ }+ }+ return types+}+// classifyStatementType returns the statement type for an omni AST node.func classifyStatementType(node ast.Node) storepb.StatementType {switch n := node.(type) {@@ -35,6 +58,16 @@ func classifyStatementType(node ast.Node) storepb.StatementType {return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIEDcase *ast.CreateEnumStmt:return storepb.StatementType_CREATE_TYPE+ case *ast.CreateTableAsStmt:+ if n.Objtype == ast.OBJECT_MATVIEW {+ return storepb.StatementType_CREATE_VIEW+ }+ return storepb.StatementType_CREATE_TABLE+ case *ast.SelectStmt:+ if hasOmniIntoClause(n) {+ return storepb.StatementType_CREATE_TABLE+ }+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED// DDL - DROPcase *ast.DropStmt:@@ -75,6 +108,8 @@ func classifyStatementType(node ast.Node) storepb.StatementType {return storepb.StatementType_UPDATEcase *ast.DeleteStmt:return storepb.StatementType_DELETE+ case *ast.MergeStmt:+ return storepb.StatementType_MERGEdefault:return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIEDdiff --git a/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.json b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.jsonnew file mode 100644index 0000000000..0a3ad18552--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.json@@ -0,0 +1,59 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Insert",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big_archive",+ "Alias": "big_archive",+ "Startup Cost": 180.00,+ "Total Cost": 280.00,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "ModifyTable",+ "Operation": "Delete",+ "Parent Relationship": "InitPlan",+ "Subplan Name": "CTE moved",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 5000,+ "Plan Width": 6,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 5000,+ "Plan Width": 6,+ "Filter": "(s_id <= 50)"+ }+ ]+ },+ {+ "Node Type": "CTE Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "CTE Name": "moved",+ "Alias": "moved",+ "Startup Cost": 0.00,+ "Total Cost": 100.00,+ "Plan Rows": 5000,+ "Plan Width": 12+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.json b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.jsonnew file mode 100644index 0000000000..840159df88--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.json@@ -0,0 +1,58 @@+[+ {+ "Plan": {+ "Node Type": "Aggregate",+ "Strategy": "Plain",+ "Partial Mode": "Simple",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 292.50,+ "Total Cost": 292.51,+ "Plan Rows": 1,+ "Plan Width": 8,+ "Plans": [+ {+ "Node Type": "ModifyTable",+ "Operation": "Delete",+ "Parent Relationship": "InitPlan",+ "Subplan Name": "CTE d",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 5000,+ "Plan Width": 6,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 5000,+ "Plan Width": 6,+ "Filter": "(s_id <= 50)"+ }+ ]+ },+ {+ "Node Type": "CTE Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "CTE Name": "d",+ "Alias": "d",+ "Startup Cost": 0.00,+ "Total Cost": 100.00,+ "Plan Rows": 5000,+ "Plan Width": 0+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.jsonnew file mode 100644index 0000000000..9510c2dd5f--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.json@@ -0,0 +1,70 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Delete",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Hash Join",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Join Type": "Inner",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 1000,+ "Plan Width": 12,+ "Inner Unique": true,+ "Hash Cond": "(big.s_id = s.id)",+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 155.00,+ "Plan Rows": 10000,+ "Plan Width": 10+ },+ {+ "Node Type": "Hash",+ "Parent Relationship": "Inner",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Filter": "flag"+ }+ ]+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.jsonnew file mode 100644index 0000000000..ba31ecf827--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.json@@ -0,0 +1,67 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Delete",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "inh_parent",+ "Alias": "inh_parent",+ "Startup Cost": 0.00,+ "Total Cost": 18.00,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Target Tables": [+ {+ "Relation Name": "inh_parent",+ "Alias": "inh_parent_1"+ },+ {+ "Relation Name": "inh_child",+ "Alias": "inh_parent_2"+ }+ ],+ "Plans": [+ {+ "Node Type": "Append",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 0.00,+ "Total Cost": 18.00,+ "Plan Rows": 101,+ "Plan Width": 10,+ "Subplans Removed": 0,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Member",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "inh_parent",+ "Alias": "inh_parent_1",+ "Startup Cost": 0.00,+ "Total Cost": 0.00,+ "Plan Rows": 1,+ "Plan Width": 10,+ "Filter": "(c < 10)"+ },+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Member",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "inh_child",+ "Alias": "inh_parent_2",+ "Startup Cost": 0.00,+ "Total Cost": 17.50,+ "Plan Rows": 100,+ "Plan Width": 10,+ "Filter": "(c < 10)"+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.jsonnew file mode 100644index 0000000000..9510c2dd5f--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.json@@ -0,0 +1,70 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Delete",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Hash Join",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Join Type": "Inner",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 1000,+ "Plan Width": 12,+ "Inner Unique": true,+ "Hash Cond": "(big.s_id = s.id)",+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 155.00,+ "Plan Rows": 10000,+ "Plan Width": 10+ },+ {+ "Node Type": "Hash",+ "Parent Relationship": "Inner",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Filter": "flag"+ }+ ]+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_select.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_select.jsonnew file mode 100644index 0000000000..e1e145deef--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_select.json@@ -0,0 +1,45 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Insert",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t2",+ "Alias": "t2",+ "Startup Cost": 5.05,+ "Total Cost": 13.30,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Bitmap Heap Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 13.30,+ "Plan Rows": 100,+ "Plan Width": 18,+ "Recheck Cond": "(c < 10)",+ "Plans": [+ {+ "Node Type": "Bitmap Index Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Index Name": "t_c_idx",+ "Startup Cost": 0.00,+ "Total Cost": 5.03,+ "Plan Rows": 100,+ "Plan Width": 0,+ "Index Cond": "(c < 10)"+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.jsonnew file mode 100644index 0000000000..ab19a70983--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.json@@ -0,0 +1,32 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Insert",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big_pk",+ "Alias": "big_pk",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Conflict Resolution": "NOTHING",+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 1000,+ "Plan Width": 12,+ "Filter": "(s_id <= 10)"+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.jsonnew file mode 100644index 0000000000..bb259bc7d1--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.json@@ -0,0 +1,31 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Insert",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big_pk",+ "Alias": "big_pk",+ "Startup Cost": 0.00,+ "Total Cost": 0.03,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Conflict Resolution": "UPDATE",+ "Conflict Arbiter Indexes": ["big_pk_pkey"],+ "Plans": [+ {+ "Node Type": "Values Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Alias": "*VALUES*",+ "Startup Cost": 0.00,+ "Total Cost": 0.03,+ "Plan Rows": 2,+ "Plan Width": 12+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/merge.json b/backend/plugin/parser/pg/test-data/explain-plan/merge.jsonnew file mode 100644index 0000000000..7fadbf86d7--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/merge.json@@ -0,0 +1,69 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Merge",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 3.25,+ "Total Cost": 185.61,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Hash Join",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Join Type": "Inner",+ "Startup Cost": 3.25,+ "Total Cost": 185.61,+ "Plan Rows": 10000,+ "Plan Width": 12,+ "Inner Unique": true,+ "Hash Cond": "(big.s_id = s.id)",+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 155.00,+ "Plan Rows": 10000,+ "Plan Width": 10+ },+ {+ "Node Type": "Hash",+ "Parent Relationship": "Inner",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.00,+ "Total Cost": 2.00,+ "Plan Rows": 100,+ "Plan Width": 10,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.00,+ "Plan Rows": 100,+ "Plan Width": 10+ }+ ]+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/select.json b/backend/plugin/parser/pg/test-data/explain-plan/select.jsonnew file mode 100644index 0000000000..3faddc7975--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/select.json@@ -0,0 +1,30 @@+[+ {+ "Plan": {+ "Node Type": "Bitmap Heap Scan",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 13.30,+ "Plan Rows": 100,+ "Plan Width": 18,+ "Recheck Cond": "(c < 10)",+ "Plans": [+ {+ "Node Type": "Bitmap Index Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Index Name": "t_c_idx",+ "Startup Cost": 0.00,+ "Total Cost": 5.03,+ "Plan Rows": 100,+ "Plan Width": 0,+ "Index Cond": "(c < 10)"+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update.json b/backend/plugin/parser/pg/test-data/explain-plan/update.jsonnew file mode 100644index 0000000000..58e0a48df0--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update.json@@ -0,0 +1,45 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 13.30,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Bitmap Heap Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 13.30,+ "Plan Rows": 100,+ "Plan Width": 38,+ "Recheck Cond": "(c < 10)",+ "Plans": [+ {+ "Node Type": "Bitmap Index Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Index Name": "t_c_idx",+ "Startup Cost": 0.00,+ "Total Cost": 5.03,+ "Plan Rows": 100,+ "Plan Width": 0,+ "Index Cond": "(c < 10)"+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.json b/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.jsonnew file mode 100644index 0000000000..0eab80be43--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.json@@ -0,0 +1,84 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 239.80,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Bitmap Heap Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Startup Cost": 5.05,+ "Total Cost": 239.80,+ "Plan Rows": 100,+ "Plan Width": 38,+ "Recheck Cond": "(c < 10)",+ "Plans": [+ {+ "Node Type": "Bitmap Index Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Index Name": "t_c_idx",+ "Startup Cost": 0.00,+ "Total Cost": 5.03,+ "Plan Rows": 100,+ "Plan Width": 0,+ "Index Cond": "(c < 10)"+ },+ {+ "Node Type": "Result",+ "Parent Relationship": "SubPlan",+ "Subplan Name": "SubPlan 2",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.25,+ "Total Cost": 2.26,+ "Plan Rows": 1,+ "Plan Width": 32,+ "Plans": [+ {+ "Node Type": "Limit",+ "Parent Relationship": "InitPlan",+ "Subplan Name": "InitPlan 1",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 0.00,+ "Total Cost": 2.25,+ "Plan Rows": 1,+ "Plan Width": 4,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.25,+ "Plan Rows": 1,+ "Plan Width": 4,+ "Filter": "(id = t.grp)"+ }+ ]+ }+ ]+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.json b/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.jsonnew file mode 100644index 0000000000..3a6d75a273--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.json@@ -0,0 +1,33 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Plans": [+ {+ "Node Type": "Bitmap Heap Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "t",+ "Alias": "t",+ "Recheck Cond": "(c < 10)",+ "Plans": [+ {+ "Node Type": "Bitmap Index Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Index Name": "t_c_idx",+ "Index Cond": "(c < 10)"+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.json b/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.jsonnew file mode 100644index 0000000000..988b5c8636--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.json@@ -0,0 +1,70 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Hash Join",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Join Type": "Inner",+ "Startup Cost": 2.12,+ "Total Cost": 184.49,+ "Plan Rows": 1000,+ "Plan Width": 16,+ "Inner Unique": true,+ "Hash Cond": "(big.s_id = s.id)",+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 155.00,+ "Plan Rows": 10000,+ "Plan Width": 10+ },+ {+ "Node Type": "Hash",+ "Parent Relationship": "Inner",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 10,+ "Filter": "flag"+ }+ ]+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.json b/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.jsonnew file mode 100644index 0000000000..997434ccb7--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.json@@ -0,0 +1,59 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 2.03,+ "Total Cost": 182.03,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Plans": [+ {+ "Node Type": "Aggregate",+ "Strategy": "Plain",+ "Partial Mode": "Simple",+ "Parent Relationship": "InitPlan",+ "Subplan Name": "InitPlan 1",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 2.02,+ "Total Cost": 2.03,+ "Plan Rows": 1,+ "Plan Width": 8,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "s",+ "Alias": "s",+ "Startup Cost": 0.00,+ "Total Cost": 2.00,+ "Plan Rows": 10,+ "Plan Width": 0,+ "Filter": "flag"+ }+ ]+ },+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "big",+ "Alias": "big",+ "Startup Cost": 0.00,+ "Total Cost": 180.00,+ "Plan Rows": 3333,+ "Plan Width": 10,+ "Filter": "(s_id <= (InitPlan 1).col1)"+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.json b/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.jsonnew file mode 100644index 0000000000..33b92b6713--- /dev/null+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.json@@ -0,0 +1,67 @@+[+ {+ "Plan": {+ "Node Type": "ModifyTable",+ "Operation": "Update",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "parted",+ "Alias": "parted",+ "Startup Cost": 0.00,+ "Total Cost": 19.00,+ "Plan Rows": 0,+ "Plan Width": 0,+ "Target Tables": [+ {+ "Relation Name": "parted_p1",+ "Alias": "parted_1"+ },+ {+ "Relation Name": "parted_p2",+ "Alias": "parted_2"+ }+ ],+ "Plans": [+ {+ "Node Type": "Append",+ "Parent Relationship": "Outer",+ "Parallel Aware": false,+ "Async Capable": false,+ "Startup Cost": 0.00,+ "Total Cost": 19.00,+ "Plan Rows": 100,+ "Plan Width": 14,+ "Subplans Removed": 0,+ "Plans": [+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Member",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "parted_p1",+ "Alias": "parted_1",+ "Startup Cost": 0.00,+ "Total Cost": 9.25,+ "Plan Rows": 50,+ "Plan Width": 14,+ "Filter": "(c < 10)"+ },+ {+ "Node Type": "Seq Scan",+ "Parent Relationship": "Member",+ "Parallel Aware": false,+ "Async Capable": false,+ "Relation Name": "parted_p2",+ "Alias": "parted_2",+ "Startup Cost": 0.00,+ "Total Cost": 9.25,+ "Plan Rows": 50,+ "Plan Width": 14,+ "Filter": "(c < 10)"+ }+ ]+ }+ ]+ }+ }+]diff --git a/backend/plugin/parser/pg/test-data/query_type.yaml b/backend/plugin/parser/pg/test-data/query_type.yamlindex 26859cd000..4b3639a899 100644--- a/backend/plugin/parser/pg/test-data/query_type.yaml+++ b/backend/plugin/parser/pg/test-data/query_type.yaml@@ -590,3 +590,114 @@results: []sourcecolumns: []predicatecolumns: []+- description: SELECT with a data-modifying CTE is DML+ statement: WITH d AS (DELETE FROM t1 WHERE a = '1' RETURNING a) SELECT count(*) FROM d;+ defaultDatabase: db+ metadata: |-+ {+ "name": "db",+ "schemas": [+ {+ "name": "public",+ "tables": [+ {+ "name": "t1",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ },+ {+ "name": "t2",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 5+ results: []+ sourcecolumns: []+ predicatecolumns: []+- description: EXPLAIN ANALYZE executes MERGE+ statement: EXPLAIN ANALYZE MERGE INTO t1 USING t2 ON t1.a = t2.a WHEN NOT MATCHED THEN INSERT (a) VALUES (t2.a);+ defaultDatabase: db+ metadata: |-+ {+ "name": "db",+ "schemas": [+ {+ "name": "public",+ "tables": [+ {+ "name": "t1",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ },+ {+ "name": "t2",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 5+ results: []+ sourcecolumns: []+ predicatecolumns: []+- description: EXPLAIN ANALYZE executes a data-modifying CTE+ statement: EXPLAIN ANALYZE WITH i AS (INSERT INTO t1 (a) VALUES ('1') RETURNING a) SELECT count(*) FROM i;+ defaultDatabase: db+ metadata: |-+ {+ "name": "db",+ "schemas": [+ {+ "name": "public",+ "tables": [+ {+ "name": "t1",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ },+ {+ "name": "t2",+ "columns": [+ {+ "name": "a",+ "type": "text"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 5+ results: []+ sourcecolumns: []+ predicatecolumns: []diff --git a/backend/plugin/parser/pg/test-data/test_statement_type.yaml b/backend/plugin/parser/pg/test-data/test_statement_type.yamlindex 17a05c94f8..712d2cd200 100644--- a/backend/plugin/parser/pg/test-data/test_statement_type.yaml+++ b/backend/plugin/parser/pg/test-data/test_statement_type.yaml@@ -129,3 +129,42 @@- statement: DROP MATERIALIZED VIEW mv1;want:- DROP_TABLE+- statement: MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;+ want:+ - MERGE+- statement: WITH d AS (DELETE FROM t1 RETURNING id) SELECT count(*) FROM d;+ want:+ - DELETE+- statement: WITH d AS (DELETE FROM t1 RETURNING *), u AS (UPDATE t3 SET c = 1 RETURNING id) INSERT INTO t2 SELECT * FROM d;+ want:+ - INSERT+ - DELETE+ - UPDATE+- statement: EXPLAIN ANALYZE MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;+ want:+ - MERGE+- statement: EXPLAIN (ANALYZE, BUFFERS) WITH d AS (DELETE FROM t1 RETURNING id) UPDATE t2 SET c = 1 FROM d;+ want:+ - UPDATE+ - DELETE+- statement: EXPLAIN WITH d AS (DELETE FROM t1 RETURNING id) SELECT count(*) FROM d;+ want: []+- statement: EXPLAIN (ANALYZE FALSE) DELETE FROM t1;+ want: []+- statement: EXPLAIN (ANALYZE 1) DELETE FROM t1;+ want:+ - DELETE+- statement: CREATE TABLE archive AS WITH d AS (DELETE FROM t1 RETURNING *) SELECT * FROM d;+ want:+ - CREATE_TABLE+ - DELETE+- statement: WITH d AS (DELETE FROM t1 RETURNING *) SELECT * INTO archive FROM d;+ want:+ - CREATE_TABLE+ - DELETE+- statement: CREATE MATERIALIZED VIEW mv AS SELECT * FROM t1;+ want:+ - CREATE_VIEW+- statement: CREATE FUNCTION purge() RETURNS bigint LANGUAGE sql BEGIN ATOMIC WITH d AS (DELETE FROM t1 RETURNING 1) SELECT count(*) FROM d; END;+ want:+ - CREATE_FUNCTIONdiff --git a/backend/plugin/parser/plsql/resource_change.go b/backend/plugin/parser/plsql/resource_change.goindex ed0efc40c0..d9d16ae0fd 100644--- a/backend/plugin/parser/plsql/resource_change.go+++ b/backend/plugin/parser/plsql/resource_change.go@@ -5,7 +5,6 @@ import (oracleast "github.com/bytebase/omni/oracle/ast"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -35,7 +34,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *modelreturn &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: extractor.sampleDMLs,+ DMLStatements: extractor.dmlStatements,DMLCount: extractor.dmlCount,InsertCount: extractor.insertCount,}, nil@@ -46,7 +45,7 @@ type omniChangedResourceExtractor struct {dbMetadata *model.DatabaseMetadatachangedResources *model.ChangedResourcesstatement string- sampleDMLs []string+ dmlStatements []stringdmlCount intinsertCount int}@@ -215,9 +214,7 @@ func (e *omniChangedResourceExtractor) addIndexTable(name *oracleast.ObjectName)func (e *omniChangedResourceExtractor) trackDML(text string) {e.dmlCount++- if len(e.sampleDMLs) < common.MaximumLintExplainSize {- e.sampleDMLs = append(e.sampleDMLs, omniStatementText(text))- }+ e.dmlStatements = append(e.dmlStatements, omniStatementText(text))}func omniStatementText(text string) string {diff --git a/backend/plugin/parser/plsql/resource_change_test.go b/backend/plugin/parser/plsql/resource_change_test.goindex f88faf6b0c..8abca22d86 100644--- a/backend/plugin/parser/plsql/resource_change_test.go+++ b/backend/plugin/parser/plsql/resource_change_test.go@@ -84,7 +84,7 @@ DELETE FROM t2 WHERE c1 = 1;`changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T2"}, false)want := &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: []string{+ DMLStatements: []string{"INSERT INTO t1 SELECT * FROM t2;","UPDATE t1 SET c1 = 5 WHERE c2 = 1;","DELETE FROM t2 WHERE c1 = 1;",@@ -153,7 +153,7 @@ END;`changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)want := &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: []string{+ DMLStatements: []string{"UPDATE t1 SET c1 = 5;",},DMLCount: 1,diff --git a/backend/plugin/parser/redshift/resource_change.go b/backend/plugin/parser/redshift/resource_change.goindex ce4babbf86..1c3c04968a 100644--- a/backend/plugin/parser/redshift/resource_change.go+++ b/backend/plugin/parser/redshift/resource_change.go@@ -6,7 +6,6 @@ import (omniredshift "github.com/bytebase/omni/redshift"redshiftast "github.com/bytebase/omni/redshift/ast"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -17,52 +16,161 @@ func init() {}func extractChangedResources(database string, schema string, dbMetadata *model.DatabaseMetadata, _ []base.AST, statement string) (*base.ChangeSummary, error) {+ if schema == "" {+ schema = "public"+ }omniSummary, err := omniredshift.ExtractChangedResources(statement, database, schema)if err != nil {return nil, err}+ stmts, err := ParseRedshift(statement)+ if err != nil {+ return nil, err+ }+ ddlAffected, err := ddlAffectedTables(stmts, database, schema)+ if err != nil {+ return nil, err+ }- changedResources := model.NewChangedResources(dbMetadata)+ summary := &base.ChangeSummary{ChangedResources: model.NewChangedResources(dbMetadata)}for _, table := range omniSummary.Tables {- changedResources.AddTable(+ summary.ChangedResources.AddTable(table.Database,table.Schema,- &storepb.ChangedResourceTable{- Name: table.Name,- },- table.Affected,+ &storepb.ChangedResourceTable{Name: table.Name},+ ddlAffected[tableKey{database: table.Database, schema: table.Schema, name: table.Name}],)}- sampleDMLs, err := extractOmniSampleDMLs(statement)- if err != nil {- return nil, err+ searchPath := schema+ // replaySearchPath is the search path set after the starting one, which the EXPLAIN connection+ // does not have.+ var replaySearchPath []string+ sampleDML := func(text string) {+ summary.DMLCount+++ text = strings.TrimSpace(text)+ if len(replaySearchPath) > 0 {+ text = base.WithSearchPath(text, replaySearchPath)+ }+ summary.DMLStatements = append(summary.DMLStatements, text)+ }+ addTable := func(rv *redshiftast.RangeVar, affectedTable bool) {+ tableDatabase, tableSchema := rv.Catalogname, rv.Schemaname+ if tableDatabase == "" {+ tableDatabase = database+ }+ if tableSchema == "" {+ tableSchema = searchPath+ }+ summary.ChangedResources.AddTable(tableDatabase, tableSchema, &storepb.ChangedResourceTable{Name: rv.Relname}, affectedTable)+ }+ for _, stmt := range stmts {+ switch n := stmt.AST.(type) {+ case *redshiftast.InsertStmt:+ if rows, ok := insertValuesRowCount(n); ok {+ summary.InsertCount += rows+ } else {+ sampleDML(stmt.Text)+ }+ case *redshiftast.UpdateStmt, *redshiftast.DeleteStmt:+ sampleDML(stmt.Text)+ case *redshiftast.MergeStmt:+ // Redshift documents EXPLAIN only for SELECT, CREATE TABLE AS, INSERT, UPDATE, and DELETE.+ summary.DMLCount+++ // omni records no MERGE target.+ if n.Relation != nil {+ addTable(n.Relation, false)+ }+ case *redshiftast.TruncateStmt:+ if n.Relations == nil {+ continue+ }+ for _, item := range n.Relations.Items {+ if rv, ok := item.(*redshiftast.RangeVar); ok {+ addTable(rv, true)+ }+ }+ case *redshiftast.VariableSetStmt:+ if schemas, ok := searchPathFromSet(n); ok {+ searchPath, replaySearchPath = schema, schemas+ if len(schemas) > 0 {+ searchPath = schemas[0]+ }+ }+ default:+ }}+ return summary, nil+}- return &base.ChangeSummary{- ChangedResources: changedResources,- SampleDMLS: sampleDMLs,- DMLCount: omniSummary.DMLCount,- InsertCount: omniSummary.InsertCount,- }, nil+type tableKey struct {+ database string+ schema string+ name string}-func extractOmniSampleDMLs(statement string) ([]string, error) {- stmts, err := ParseRedshift(statement)+// ddlAffectedTables returns the tables whose existing rows the script's DDL affects.+// omni also flags DML targets and merges flags across statements, so the script it+// reads here leaves out the DML.+func ddlAffectedTables(stmts []omniredshift.Statement, database, schema string) (map[tableKey]bool, error) {+ var texts []string+ for _, stmt := range stmts {+ switch stmt.AST.(type) {+ case *redshiftast.InsertStmt, *redshiftast.UpdateStmt, *redshiftast.DeleteStmt, *redshiftast.MergeStmt:+ default:+ texts = append(texts, stmt.Text)+ }+ }+ // A statement's text lacks its semicolon when a comment precedes the semicolon.+ summary, err := omniredshift.ExtractChangedResources(strings.Join(texts, ";\n"), database, schema)if err != nil {return nil, err}+ affected := make(map[tableKey]bool)+ for _, table := range summary.Tables {+ if table.Affected {+ affected[tableKey{database: table.Database, schema: table.Schema, name: table.Name}] = true+ }+ }+ return affected, nil+}- var sampleDMLs []string- for _, stmt := range stmts {- if len(sampleDMLs) >= common.MaximumLintExplainSize {- break+// insertValuesRowCount returns how many rows INSERT ... VALUES or INSERT ... DEFAULT+// VALUES adds, and false for an INSERT whose rows come from a query.+func insertValuesRowCount(n *redshiftast.InsertStmt) (int, bool) {+ if n.SelectStmt == nil {+ return 1, true+ }+ if s, ok := n.SelectStmt.(*redshiftast.SelectStmt); ok && s.ValuesLists != nil {+ return len(s.ValuesLists.Items), true+ }+ return 0, false+}++// searchPathFromSet returns the schemas a SET search_path names, skipping "$user", or none for RESET+// or SET ... TO DEFAULT, which restore the starting search path. ok is false for any other statement.+// Like omni's ExtractChangedResources, unqualified names resolve to the first schema.+func searchPathFromSet(n *redshiftast.VariableSetStmt) ([]string, bool) {+ if !strings.EqualFold(n.Name, "search_path") {+ return nil, false+ }+ if n.Kind == redshiftast.VAR_SET_DEFAULT || n.Kind == redshiftast.VAR_RESET {+ return nil, true+ }+ if n.Args == nil {+ return nil, false+ }+ var schemas []string+ for _, arg := range n.Args.Items {+ c, ok := arg.(*redshiftast.A_Const)+ if !ok {+ continue}- switch stmt.AST.(type) {- case *redshiftast.InsertStmt, *redshiftast.UpdateStmt, *redshiftast.DeleteStmt, *redshiftast.MergeStmt:- sampleDMLs = append(sampleDMLs, strings.TrimSpace(stmt.Text))- default:+ s, ok := c.Val.(*redshiftast.String)+ if !ok || s.Str == "" || s.Str == "$user" {+ continue}+ schemas = append(schemas, s.Str)}- return sampleDMLs, nil+ return schemas, len(schemas) > 0}diff --git a/backend/plugin/parser/redshift/resource_change_test.go b/backend/plugin/parser/redshift/resource_change_test.goindex 037775d939..1336a4e752 100644--- a/backend/plugin/parser/redshift/resource_change_test.go+++ b/backend/plugin/parser/redshift/resource_change_test.go@@ -1,6 +1,8 @@package redshiftimport (+ "slices"+ "strings""testing"metadatapb "github.com/bytebase/omni/metadata"@@ -12,83 +14,149 @@ import ()func TestExtractChangedResources(t *testing.T) {- statement := `CREATE TABLE t1 (c1 INT);+ type table struct {+ database string+ schema string+ name string+ affected bool+ }+ for _, tc := range []struct {+ name string+ statement string+ tables []table+ dmlStatements []string+ dmlCount int+ insertCount int+ }{+ {+ name: "ddl_affects_a_table_that_dml_also_changes",+ statement: `CREATE TABLE t1 (c1 INT);ALTER TABLE t1 ADD COLUMN c2 INT;INSERT INTO t1 VALUES (1);-UPDATE t1 SET c2 = 2;`-- dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)- changedResources := model.NewChangedResources(dbMetadata)- changedResources.AddTable(- "db",- "public",- &storepb.ChangedResourceTable{- Name: "t1",+UPDATE t1 SET c2 = 2;`,+ tables: []table{{database: "db", schema: "public", name: "t1", affected: true}},+ dmlStatements: []string{"UPDATE t1 SET c2 = 2;"},+ dmlCount: 1,+ insertCount: 1,},- true,- )- want := &base.ChangeSummary{- ChangedResources: changedResources,- SampleDMLS: []string{- "INSERT INTO t1 VALUES (1);",- "UPDATE t1 SET c2 = 2;",+ {+ name: "dml_targets_are_not_affected",+ statement: `UPDATE t SET c = 1 WHERE id = 1;+DELETE FROM t WHERE id = 2;`,+ tables: []table{{database: "db", schema: "public", name: "t"}},+ dmlStatements: []string{+ "UPDATE t SET c = 1 WHERE id = 1;",+ "DELETE FROM t WHERE id = 2;",+ },+ dmlCount: 2,},- DMLCount: 2,- InsertCount: 1,- }-- stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, statement)- require.NoError(t, err)- got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), statement)- require.NoError(t, err)- require.Equal(t, want, got)-}--func TestExtractChangedResourcesSearchPathDropAndSelectInto(t *testing.T) {- statement := `SET search_path TO analytics;+ {+ name: "insert_values_counts_rows_without_a_sample",+ statement: `INSERT INTO t VALUES (1), (2), (3);+INSERT INTO t (c) VALUES (4);+INSERT INTO t DEFAULT VALUES;`,+ tables: []table{{database: "db", schema: "public", name: "t"}},+ insertCount: 5,+ },+ {+ name: "insert_select_is_sampled",+ statement: `INSERT INTO t SELECT * FROM s WHERE c > 1;`,+ tables: []table{{database: "db", schema: "public", name: "t"}},+ dmlStatements: []string{"INSERT INTO t SELECT * FROM s WHERE c > 1;"},+ dmlCount: 1,+ },+ {+ name: "merge_is_counted_without_a_sample",+ statement: `MERGE INTO m USING s ON m.id = s.id WHEN MATCHED THEN UPDATE SET c = s.c;+UPDATE t SET c = 1 WHERE id = 1;`,+ tables: []table{{database: "db", schema: "public", name: "m"}, {database: "db", schema: "public", name: "t"}},+ dmlStatements: []string{"UPDATE t SET c = 1 WHERE id = 1;"},+ dmlCount: 2,+ },+ {+ name: "truncate_affects_tables_resolved_against_the_search_path",+ statement: `TRUNCATE a;+SET search_path TO analytics;+TRUNCATE TABLE b, public.c, other.s.d;+RESET search_path;+TRUNCATE e;`,+ tables: []table{+ {database: "db", schema: "public", name: "a", affected: true},+ {database: "db", schema: "analytics", name: "b", affected: true},+ {database: "db", schema: "public", name: "c", affected: true},+ {database: "other", schema: "s", name: "d", affected: true},+ {database: "db", schema: "public", name: "e", affected: true},+ },+ },+ {+ name: "search_path_drop_and_select_into",+ statement: `SET search_path TO analytics;CREATE TABLE unqualified(id INT);SELECT * INTO copied_rows FROM public.rows;DROP TABLE old_rows;-DELETE FROM copied_rows WHERE id = 1;`-- dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)- changedResources := model.NewChangedResources(dbMetadata)- changedResources.AddTable(- "db",- "analytics",- &storepb.ChangedResourceTable{- Name: "unqualified",+DELETE FROM copied_rows WHERE id = 1;`,+ tables: []table{+ {database: "db", schema: "analytics", name: "unqualified"},+ {database: "db", schema: "analytics", name: "copied_rows"},+ {database: "db", schema: "analytics", name: "old_rows", affected: true},+ },+ dmlStatements: []string{"SET LOCAL search_path TO \"analytics\";\nDELETE FROM copied_rows WHERE id = 1;"},+ dmlCount: 1,},- false,- )- changedResources.AddTable(- "db",- "analytics",- &storepb.ChangedResourceTable{- Name: "copied_rows",+ {+ name: "sampled_dml_replays_the_search_path",+ statement: `SET search_path TO '$user', analytics, public;+UPDATE t SET c = 1;+RESET search_path;+UPDATE t SET c = 2;`,+ tables: []table{+ {database: "db", schema: "analytics", name: "t"},+ {database: "db", schema: "public", name: "t"},+ },+ dmlStatements: []string{+ "SET LOCAL search_path TO \"analytics\", \"public\";\nUPDATE t SET c = 1;",+ "UPDATE t SET c = 2;",+ },+ dmlCount: 2,},- true,- )- changedResources.AddTable(- "db",- "analytics",- &storepb.ChangedResourceTable{- Name: "old_rows",+ {+ name: "comment_before_semicolon_between_ddl_and_dml",+ statement: `ALTER TABLE a ADD COLUMN c INT /* note */;+INSERT INTO b VALUES (1);+DROP TABLE c;`,+ tables: []table{+ {database: "db", schema: "public", name: "a", affected: true},+ {database: "db", schema: "public", name: "b"},+ {database: "db", schema: "public", name: "c", affected: true},+ },+ insertCount: 1,},- true,- )- want := &base.ChangeSummary{- ChangedResources: changedResources,- SampleDMLS: []string{- "DELETE FROM copied_rows WHERE id = 1;",+ {+ name: "every_dml_statement_is_listed",+ statement: strings.Repeat("UPDATE t SET c = 1;\n", 12),+ tables: []table{{database: "db", schema: "public", name: "t"}},+ dmlStatements: slices.Repeat([]string{"UPDATE t SET c = 1;"}, 12),+ dmlCount: 12,},- DMLCount: 1,- InsertCount: 0,- }+ } {+ t.Run(tc.name, func(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)+ changedResources := model.NewChangedResources(dbMetadata)+ for _, table := range tc.tables {+ changedResources.AddTable(table.database, table.schema, &storepb.ChangedResourceTable{Name: table.name}, table.affected)+ }+ want := &base.ChangeSummary{+ ChangedResources: changedResources,+ DMLStatements: tc.dmlStatements,+ DMLCount: tc.dmlCount,+ InsertCount: tc.insertCount,+ }- stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, statement)- require.NoError(t, err)- got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), statement)- require.NoError(t, err)- require.Equal(t, want, got)+ stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, tc.statement)+ require.NoError(t, err)+ got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)+ require.NoError(t, err)+ require.Equal(t, want, got)+ })+ }}diff --git a/backend/plugin/parser/redshift/statement_type.go b/backend/plugin/parser/redshift/statement_type.goindex 526f9b3b24..c4664fdff4 100644--- a/backend/plugin/parser/redshift/statement_type.go+++ b/backend/plugin/parser/redshift/statement_type.go@@ -115,6 +115,8 @@ func classifyOmniStatementType(node redshiftast.Node) storepb.StatementType {return storepb.StatementType_UPDATEcase *redshiftast.DeleteStmt:return storepb.StatementType_DELETE+ case *redshiftast.MergeStmt:+ return storepb.StatementType_MERGEcase *redshiftast.RedshiftObjectStmt:return classifyOmniRedshiftObjectStatementType(n.Command, n.ObjectType)default:diff --git a/backend/plugin/parser/redshift/statement_type_test.go b/backend/plugin/parser/redshift/statement_type_test.goindex 813aca0fe6..3d52ff5f9e 100644--- a/backend/plugin/parser/redshift/statement_type_test.go+++ b/backend/plugin/parser/redshift/statement_type_test.go@@ -120,6 +120,13 @@ func TestGetStatementTypes(t *testing.T) {storepb.StatementType_DELETE,},},+ {+ name: "merge",+ statement: "MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;",+ want: []storepb.StatementType{+ storepb.StatementType_MERGE,+ },+ },{name: "truncate table",statement: "TRUNCATE TABLE t1;",diff --git a/backend/plugin/parser/tidb/resource_change.go b/backend/plugin/parser/tidb/resource_change.goindex d5f3d3ec38..a5c07a6e09 100644--- a/backend/plugin/parser/tidb/resource_change.go+++ b/backend/plugin/parser/tidb/resource_change.go@@ -4,10 +4,10 @@ import ("strings""unicode"+ omniast "github.com/bytebase/omni/tidb/ast"tidbast "github.com/pingcap/tidb/pkg/parser/ast""github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -19,17 +19,24 @@ func init() {}func extractChangedResources(database string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {- changedResources := model.NewChangedResources(dbMetadata)- dmlCount := 0- insertCount := 0- var sampleDMLs []string+ summary := &base.ChangeSummary{+ ChangedResources: model.NewChangedResources(dbMetadata),+ }for _, ast := range asts {tidbAST, ok := GetTiDBAST(ast)if !ok {- return nil, errors.New("expected TiDB AST")+ // The pingcap re-parse rejects some statements omni accepts. Read those from the+ // omni AST rather than failing the summary of every other statement.+ omniAST, ok := ast.(*OmniAST)+ if !ok {+ return nil, errors.New("expected TiDB AST")+ }+ addOmniTables(database, omniAST.Node, summary.ChangedResources)+ countOmniDML(summary, omniAST)+ continue}node := tidbAST.Node- err := getResourceChanges(database, node, statement, changedResources)+ err := getResourceChanges(database, node, statement, summary.ChangedResources)if err != nil {return nil, err}@@ -37,29 +44,128 @@ func extractChangedResources(database string, _ string, dbMetadata *model.Databaswitch n := node.(type) {case *tidbast.InsertStmt:if len(n.Lists) > 0 {- insertCount += len(n.Lists)+ summary.InsertCount += len(n.Lists)continue}-- dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, trimStatement(n.Text()))- }+ addSampleDML(summary, n.Text())case *tidbast.UpdateStmt, *tidbast.DeleteStmt:- dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, trimStatement(node.Text()))+ addSampleDML(summary, node.Text())+ case *tidbast.NonTransactionalDMLStmt:+ if n.DryRun == tidbast.NoDryRun {+ addBatchSampleDML(summary, ast)}default:}}+ return summary, nil+}- return &base.ChangeSummary{- ChangedResources: changedResources,- DMLCount: dmlCount,- SampleDMLS: sampleDMLs,- InsertCount: insertCount,- }, nil+// addOmniTables records the tables an omni INSERT, UPDATE, DELETE, or BATCH statement names, as+// getResourceChanges does for the pingcap AST.+func addOmniTables(database string, node omniast.Node, changedResources *model.ChangedResources) {+ var tableExprs []omniast.TableExpr+ switch n := node.(type) {+ case *omniast.InsertStmt:+ if n.Table != nil {+ tableExprs = append(tableExprs, n.Table)+ }+ case *omniast.UpdateStmt:+ tableExprs = n.Tables+ case *omniast.DeleteStmt:+ tableExprs = append(append(tableExprs, n.Tables...), n.Using...)+ case *omniast.BatchStmt:+ if n.DryRun == omniast.BatchDryRunNone {+ addOmniTables(database, n.DML, changedResources)+ }+ return+ default:+ return+ }+ var add func(expr omniast.TableExpr)+ add = func(expr omniast.TableExpr) {+ switch e := expr.(type) {+ case *omniast.TableRef:+ d := e.Schema+ if d == "" {+ d = database+ }+ changedResources.AddTable(d, "", &storepb.ChangedResourceTable{Name: e.Name}, false)+ case *omniast.JoinClause:+ add(e.Left)+ add(e.Right)+ default:+ }+ }+ for _, expr := range tableExprs {+ add(expr)+ }+}++func countOmniDML(summary *base.ChangeSummary, omniAST *OmniAST) {+ switch n := omniAST.Node.(type) {+ case *omniast.InsertStmt:+ switch {+ case len(n.Values) > 0:+ summary.InsertCount += len(n.Values)+ case len(n.SetList) > 0:+ summary.InsertCount+++ default:+ addSampleDML(summary, omniAST.Text)+ }+ case *omniast.UpdateStmt, *omniast.DeleteStmt:+ addSampleDML(summary, omniAST.Text)+ case *omniast.BatchStmt:+ if n.DryRun == omniast.BatchDryRunNone {+ addBatchSampleDML(summary, omniAST)+ }+ default:+ }+}++// addBatchSampleDML samples the DML inside a BATCH statement, because EXPLAIN rejects the+// BATCH syntax. A BATCH without an omni AST is counted but not sampled.+func addBatchSampleDML(summary *base.ChangeSummary, ast base.AST) {+ omniAST, ok := ast.(*OmniAST)+ if !ok {+ summary.DMLCount+++ return+ }+ batch, ok := omniAST.Node.(*omniast.BatchStmt)+ if !ok {+ summary.DMLCount+++ return+ }+ // omni positions the text after an executable comment as if the comment's markers were removed.+ if strings.Contains(omniAST.Text, "/*!") || strings.Contains(omniAST.Text, "/*T!") || strings.Contains(omniAST.Text, "/*M!") {+ summary.DMLCount+++ return+ }+ var loc omniast.Loc+ switch dml := batch.DML.(type) {+ case *omniast.DeleteStmt:+ loc = dml.Loc+ case *omniast.UpdateStmt:+ loc = dml.Loc+ case *omniast.InsertStmt:+ loc = dml.Loc+ default:+ summary.DMLCount+++ return+ }+ end := loc.End+ if end <= loc.Start || end > len(omniAST.Text) {+ end = len(omniAST.Text)+ }+ if loc.Start <= 0 || loc.Start >= end {+ summary.DMLCount+++ return+ }+ addSampleDML(summary, omniAST.Text[loc.Start:end])+}++func addSampleDML(summary *base.ChangeSummary, text string) {+ summary.DMLCount+++ summary.DMLStatements = append(summary.DMLStatements, trimStatement(text))}// addTiDBObjectDatabase records a database-only write target for a non-table object DDL@@ -293,6 +399,10 @@ func getResourceChanges(database string, node tidbast.StmtNode, _ string, change)}}+ case *tidbast.NonTransactionalDMLStmt:+ if node.DryRun == tidbast.NoDryRun {+ return getResourceChanges(database, node.DMLStmt, "", changedResources)+ }default:}return nildiff --git a/backend/plugin/parser/tidb/resource_change_test.go b/backend/plugin/parser/tidb/resource_change_test.goindex 056eb88ae8..041236ef16 100644--- a/backend/plugin/parser/tidb/resource_change_test.go+++ b/backend/plugin/parser/tidb/resource_change_test.go@@ -37,7 +37,7 @@ func TestExtractChangedResources(t *testing.T) {)want := &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: []string{+ DMLStatements: []string{"UPDATE t1 SET c1 = 5",},DMLCount: 1,@@ -106,3 +106,66 @@ func TestExtractChangedResourcesObjectDDLDatabaseOnly(t *testing.T) {require.NoError(t, err)require.Empty(t, got.ChangedResources.GetDatabaseOnlyTargets())}++func TestExtractChangedResourcesDMLCounts(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ dmlCount int+ insertCount int+ samples []string+ tables []string+ }{+ {+ name: "insert_set_is_one_row",+ statement: `INSERT INTO t SET c = 1, d = 2;`,+ insertCount: 1,+ tables: []string{"t"},+ },+ {+ name: "batch_samples_inner_dml",+ statement: `BATCH ON id LIMIT 1000 DELETE FROM t WHERE c = 1;`,+ dmlCount: 1,+ samples: []string{"DELETE FROM t WHERE c = 1"},+ tables: []string{"t"},+ },+ {+ name: "batch_with_an_executable_comment_is_counted_without_a_sample",+ statement: `BATCH ON id LIMIT 1000 DELETE FROM t WHERE /*T![clustered_index] c = 0 AND */ id < 10000000000;`,+ dmlCount: 1,+ tables: []string{"t"},+ },+ {+ name: "batch_dry_run_changes_nothing",+ statement: `BATCH ON id LIMIT 1000 DRY RUN DELETE FROM t WHERE c = 1;`,+ },+ {+ // pingcap cannot parse the row alias, which omni accepts.+ name: "statement_without_pingcap_ast",+ statement: "INSERT INTO t VALUES (1, 2) AS new ON DUPLICATE KEY UPDATE c = new.c;\nUPDATE t2 SET c = 1 WHERE id = 1;",+ dmlCount: 1,+ insertCount: 1,+ samples: []string{"UPDATE t2 SET c = 1 WHERE id = 1"},+ tables: []string{"t", "t2"},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ stmts, err := base.ParseStatements(storepb.Engine_TIDB, tc.statement)+ require.NoError(t, err)+ got, err := extractChangedResources("db", "", nil /* dbMetadata */, base.ExtractASTs(stmts), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.dmlCount, got.DMLCount)+ require.Equal(t, tc.insertCount, got.InsertCount)+ require.Equal(t, tc.samples, got.DMLStatements)+ var tables []string+ for _, database := range got.ChangedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ tables = append(tables, table.GetName())+ }+ }+ }+ require.Equal(t, tc.tables, tables)+ })+ }+}diff --git a/backend/plugin/parser/tsql/resource_change.go b/backend/plugin/parser/tsql/resource_change.goindex 08ebd6af4c..b39ed043f0 100644--- a/backend/plugin/parser/tsql/resource_change.go+++ b/backend/plugin/parser/tsql/resource_change.go@@ -7,7 +7,6 @@ import ("github.com/bytebase/omni/mssql/ast""github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model"@@ -22,13 +21,11 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMetchangedResources := model.NewChangedResources(dbMetadata)var dmlCount, insertCount int- var sampleDMLs []string+ var dmlStatements []stringaddDML := func(text string) {dmlCount++- if len(sampleDMLs) < common.MaximumLintExplainSize {- sampleDMLs = append(sampleDMLs, trimStatement(text))- }+ dmlStatements = append(dmlStatements, trimStatement(text))}addTable := func(ref *ast.TableRef, affectData bool) {@@ -116,6 +113,12 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMetinsertCount += vc.Rows.Len()continue}+ // SHOWPLAN estimates one row for INSERT ... EXEC whatever the procedure returns, so the+ // statement is counted without an estimate.+ if _, ok := n.Source.(*ast.ExecStmt); ok {+ dmlCount+++ continue+ }addDML(omniAST.Text)case *ast.UpdateStmt:@@ -170,7 +173,7 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMetreturn &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: sampleDMLs,+ DMLStatements: dmlStatements,DMLCount: dmlCount,InsertCount: insertCount,}, nildiff --git a/backend/plugin/parser/tsql/resource_change_test.go b/backend/plugin/parser/tsql/resource_change_test.goindex 9a7ddab9e8..502c78bba8 100644--- a/backend/plugin/parser/tsql/resource_change_test.go+++ b/backend/plugin/parser/tsql/resource_change_test.go@@ -28,7 +28,7 @@ func TestExtractChangedResources(t *testing.T) {)want := &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: []string{+ DMLStatements: []string{"UPDATE t1 SET c1 = 5;",},DMLCount: 1,@@ -68,7 +68,20 @@ func TestExtractChangedResources_InsertDefaultValues(t *testing.T) {require.NoError(t, err)require.Equal(t, 1, got.InsertCount)require.Equal(t, 0, got.DMLCount)- require.Empty(t, got.SampleDMLS)+ require.Empty(t, got.DMLStatements)+}++func TestExtractChangedResources_InsertExec(t *testing.T) {+ const statement = `INSERT INTO dbo.t1 EXEC dbo.sp;`++ stmts, err := base.ParseStatements(storepb.Engine_MSSQL, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ got, err := extractChangedResources("DB", "dbo", nil, asts, statement)+ require.NoError(t, err)+ require.Equal(t, 1, got.DMLCount)+ require.Empty(t, got.DMLStatements)+ require.Zero(t, got.InsertCount)}func TestExtractChangedResources_Truncate(t *testing.T) {diff --git a/backend/runner/plancheck/statement_report_executor.go b/backend/runner/plancheck/statement_report_executor.goindex 2d66c6f731..041b8ad260 100644--- a/backend/runner/plancheck/statement_report_executor.go+++ b/backend/runner/plancheck/statement_report_executor.go@@ -4,6 +4,7 @@ import ("context""fmt""log/slog"+ "strings""github.com/pkg/errors"@@ -11,7 +12,9 @@ import ("github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/dbfactory""github.com/bytebase/bytebase/backend/component/sheet"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code""github.com/bytebase/bytebase/backend/plugin/db"+ cockroachdbdriver "github.com/bytebase/bytebase/backend/plugin/db/cockroachdb"mssqldriver "github.com/bytebase/bytebase/backend/plugin/db/mssql"mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"oracledriver "github.com/bytebase/bytebase/backend/plugin/db/oracle"@@ -100,7 +103,7 @@ func (e *StatementReportExecutor) RunForTarget(ctx context.Context, target *ChecCode: common.Ok.Int32(),Title: "OK",}- summaryReport, err := GetSQLSummaryReport(ctx, e.store, e.sheetManager, e.dbFactory, database, fullSheet.Statement)+ summaryReport, estimateWarning, err := GetSQLSummaryReport(ctx, e.store, e.sheetManager, e.dbFactory, database, fullSheet.Statement)if err != nil {return nil, err}@@ -109,17 +112,28 @@ func (e *StatementReportExecutor) RunForTarget(ctx context.Context, target *ChecSqlSummaryReport: summaryReport,}}+ if estimateWarning != "" {+ planCheckRunResult.Status = storepb.Advice_WARNING+ planCheckRunResult.Code = code.StatementExplainQueryFailed.Int32()+ planCheckRunResult.Title = AffectedRowsEstimateIncompleteTitle+ planCheckRunResult.Content = estimateWarning+ }return []*storepb.PlanCheckRunResult_Result{planCheckRunResult}, nil}-// GetSQLSummaryReport gets the SQL summary report for the given statement and database.-func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager *sheet.Manager, dbFactory *dbfactory.DBFactory, database *store.DatabaseMessage, statement string) (*storepb.PlanCheckRunResult_Result_SqlSummaryReport, error) {+// AffectedRowsEstimateIncompleteTitle titles the warning for DML statements whose affected+// rows could not be estimated.+const AffectedRowsEstimateIncompleteTitle = "Affected rows estimate is incomplete"++// GetSQLSummaryReport gets the SQL summary report for the given statement and database. The+// returned warning is non-empty when some DML statements' affected rows could not be estimated.+func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager *sheet.Manager, dbFactory *dbfactory.DBFactory, database *store.DatabaseMessage, statement string) (*storepb.PlanCheckRunResult_Result_SqlSummaryReport, string, error) {instance, err := stores.GetInstanceByResourceID(ctx, database.InstanceID)if err != nil {- return nil, err+ return nil, "", err}if instance == nil {- return nil, errors.Errorf("instance not found: %s", database.InstanceID)+ return nil, "", errors.Errorf("instance not found: %s", database.InstanceID)}databaseSchema, err := stores.GetDBSchema(ctx, &store.FindDBSchemaMessage{Workspace: instance.Workspace,@@ -127,19 +141,19 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManagerDatabaseName: database.DatabaseName,})if err != nil {- return nil, err+ return nil, "", err}if databaseSchema == nil {- return nil, errors.Errorf("database schema %s not found", database.String())+ return nil, "", errors.Errorf("database schema %s not found", database.String())}if databaseSchema.GetProto() == nil {- return nil, errors.Errorf("database schema metadata %s not found", database.String())+ return nil, "", errors.Errorf("database schema metadata %s not found", database.String())}stmts, syntaxAdvices := sheetManager.GetStatementsForChecks(instance.Metadata.GetEngine(), statement)if len(syntaxAdvices) > 0 {// Return nil as it should already be checked before running this function.- return nil, nil+ return nil, "", nil}asts := parserbase.ExtractASTs(stmts)@@ -147,13 +161,13 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManagervar defaultSchema stringproject, err := stores.GetProject(ctx, &store.FindProjectMessage{Workspace: instance.Workspace, ResourceID: &database.ProjectID})if err != nil {- return nil, err+ return nil, "", err}driver, err := dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),})if err != nil {- return nil, err+ return nil, "", err}defer driver.Close(ctx)@@ -161,7 +175,7 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManagercase storepb.Engine_POSTGRES:pd, ok := driver.(*pgdriver.Driver)if !ok {- return nil, errors.Errorf("invalid pg driver type")+ return nil, "", errors.Errorf("invalid pg driver type")}explainCalculator = pd.CountAffectedRows// Empty so pg's extractChangedResources falls back to the database's actual@@ -170,46 +184,54 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManagercase storepb.Engine_REDSHIFT:rd, ok := driver.(*redshiftdriver.Driver)if !ok {- return nil, errors.Errorf("invalid redshift driver type")+ return nil, "", errors.Errorf("invalid redshift driver type")}explainCalculator = rd.CountAffectedRowsdefaultSchema = "public"+ case storepb.Engine_COCKROACHDB:+ cd, ok := driver.(*cockroachdbdriver.Driver)+ if !ok {+ return nil, "", errors.Errorf("invalid cockroachdb driver type")+ }+ explainCalculator = cd.CountAffectedRows+ // Empty so the extractor resolves names with the database's synced search_path, as for PostgreSQL.+ defaultSchema = ""case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:md, ok := driver.(*mysqldriver.Driver)if !ok {- return nil, errors.Errorf("invalid mysql driver type")+ return nil, "", errors.Errorf("invalid mysql driver type")}explainCalculator = md.CountAffectedRowsdefaultSchema = ""case storepb.Engine_TIDB:md, ok := driver.(*tidbdriver.Driver)if !ok {- return nil, errors.Errorf("invalid tidb driver type")+ return nil, "", errors.Errorf("invalid tidb driver type")}explainCalculator = md.CountAffectedRowsdefaultSchema = ""case storepb.Engine_ORACLE:od, ok := driver.(*oracledriver.Driver)if !ok {- return nil, errors.Errorf("invalid oracle driver type")+ return nil, "", errors.Errorf("invalid oracle driver type")}explainCalculator = od.CountAffectedRowsdefaultSchema = database.DatabaseNamecase storepb.Engine_MSSQL:md, ok := driver.(*mssqldriver.Driver)if !ok {- return nil, errors.Errorf("invalid mssql driver type")+ return nil, "", errors.Errorf("invalid mssql driver type")}explainCalculator = md.CountAffectedRowsdefaultSchema = "dbo"default:// Already checked in the Run().- return nil, nil+ return nil, "", nil}sqlTypes, err := SummaryStatementTypes(instance.Metadata.GetEngine(), asts)if err != nil {- return nil, err+ return nil, "", err}// Database secrets feature has been removed@@ -217,15 +239,15 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager// Database secrets feature removed - using original statement directlychangeSummary, err := parserbase.ExtractChangedResources(instance.Metadata.GetEngine(), database.DatabaseName, defaultSchema, databaseSchema, asts, statement)if err != nil {- return nil, errors.Wrapf(err, "failed to extract changed resources")+ return nil, "", errors.Wrapf(err, "failed to extract changed resources")}- totalAffectedRows := calculateAffectedRows(ctx, changeSummary, explainCalculator)+ totalAffectedRows, estimateWarning := calculateAffectedRows(ctx, instance.Metadata.GetEngine(), changeSummary, explainCalculator)return &storepb.PlanCheckRunResult_Result_SqlSummaryReport{StatementTypes: sqlTypes,AffectedRows: totalAffectedRows,ChangedResources: changeSummary.ChangedResources.Build(),- }, nil+ }, estimateWarning, nil}// SummaryStatementTypes classifies a summary report's statements through the@@ -244,27 +266,326 @@ func SummaryStatementTypes(engine storepb.Engine, asts []parserbase.AST) ([]stortype getAffectedRowsFromExplain func(context.Context, string) (int64, error)-func calculateAffectedRows(ctx context.Context, changeSummary *parserbase.ChangeSummary, explainCalculator getAffectedRowsFromExplain) int64 {- var totalAffectedRows int64- // Count DMLs.- sampleCount := 0+// calculateAffectedRows adds the estimated rows of the DML statements, the inserted VALUES rows,+// and the rows of tables changed by DDL. The returned warning describes DML statements whose rows+// could not be estimated.+func calculateAffectedRows(ctx context.Context, engine storepb.Engine, changeSummary *parserbase.ChangeSummary, explainCalculator getAffectedRowsFromExplain) (int64, string) {+ mysqlFamily := engine == storepb.Engine_MYSQL || engine == storepb.Engine_MARIADB || engine == storepb.Engine_TIDB || engine == storepb.Engine_OCEANBASE+ shapes := groupStatementsByShape(changeSummary.DMLStatements, mysqlFamily)+ sampled := 0+ var failures []errorif explainCalculator != nil {- for _, dml := range changeSummary.SampleDMLS {- count, err := explainCalculator(ctx, dml)- if err != nil {- slog.Error("failed to calculate affected rows", log.BBError(err))- continue+ // Each round estimates the next statement of every shape, so the samples cover as many+ // shapes as they can before sampling a shape twice.+ for round := 0; sampled < common.MaximumLintExplainSize; round++ {+ sampledInRound := false+ for _, shape := range shapes {+ if round >= len(shape.statements) || sampled >= common.MaximumLintExplainSize {+ continue+ }+ sampledInRound = true+ sampled+++ count, err := explainCalculator(ctx, shape.statements[round])+ if err != nil {+ slog.Error("failed to calculate affected rows", log.BBError(err))+ failures = append(failures, err)+ shape.failed+++ continue+ }+ shape.rows = common.AddRows(shape.rows, count)+ shape.estimated++}- sampleCount++- totalAffectedRows += count+ if !sampledInRound {+ break+ }+ }+ }++ var dmlRows, estimatedRows int64+ estimated := 0+ // The unestimated statements of a sampled shape count as that shape's average estimate; the+ // statements of other shapes, and DML statements without text, count as the overall average and+ // are reported as not estimated.+ unestimated := max(changeSummary.DMLCount-len(changeSummary.DMLStatements), 0)+ for _, shape := range shapes {+ estimatedRows = common.AddRows(estimatedRows, shape.rows)+ estimated += shape.estimated+ switch {+ case shape.estimated == len(shape.statements):+ dmlRows = common.AddRows(dmlRows, shape.rows)+ case shape.estimated > 0:+ dmlRows = common.AddRows(dmlRows, common.RoundRows(float64(shape.rows)/float64(shape.estimated)*float64(len(shape.statements))))+ default:+ unestimated += len(shape.statements)+ }+ }+ if estimated > 0 && unestimated > 0 {+ dmlRows = common.AddRows(dmlRows, common.RoundRows(float64(estimatedRows)/float64(estimated)*float64(unestimated)))+ }+ totalAffectedRows := common.AddRows(dmlRows, int64(changeSummary.InsertCount))+ totalAffectedRows = common.AddRows(totalAffectedRows, changeSummary.ChangedResources.CountAffectedTableRows())++ // A failed sample of a shape with other estimates counts as that shape's average, but is still+ // not estimated.+ notEstimated := unestimated+ for _, shape := range shapes {+ if shape.estimated > 0 {+ notEstimated += shape.failed+ }+ }+ var warning string+ switch {+ case len(failures) > 0:+ warning = fmt.Sprintf("Affected rows could not be estimated for %d of %d DML statements: %v", notEstimated, changeSummary.DMLCount, failures[0])+ case notEstimated > 0:+ warning = fmt.Sprintf("Affected rows could not be estimated for %d of %d DML statements.", notEstimated, changeSummary.DMLCount)+ default:+ }+ return totalAffectedRows, warning+}++// statementShape is the DML statements that share a shape, with the estimates of those sampled.+type statementShape struct {+ statements []string+ estimated int+ failed int+ rows int64+}++// groupStatementsByShape groups statements that differ only in literal values or spacing, in the+// order each shape first appears.+func groupStatementsByShape(statements []string, mysqlFamily bool) []*statementShape {+ var shapes []*statementShape+ byKey := map[string]*statementShape{}+ for _, statement := range statements {+ key := shapeKey(statement, mysqlFamily)+ shape, ok := byKey[key]+ if !ok {+ shape = &statementShape{}+ byKey[key] = shape+ shapes = append(shapes, shape)}+ shape.statements = append(shape.statements, statement)}- if sampleCount > 0 {- totalAffectedRows = int64((float64(totalAffectedRows) / float64(sampleCount)) * float64(changeSummary.DMLCount))+ return shapes+}++// shapeKey replaces the string and numeric literals of a statement with ? and drops comments and+// whitespace that does not separate words. Everything else keeps its text, including identifiers,+// their case, double-quoted text, which MySQL's ANSI_QUOTES makes an identifier, and the hints and+// executable comments that isKeptComment lists. A statement with text the scan does not follow, such+// as an unclosed quote, an Oracle q'{...}' string, a dollar-quoted string, a nested comment, or a+// quote in brackets, is its own key, because a quote in that text could make the scan read the+// statement's clauses as a literal. mysqlFamily applies the MySQL, MariaDB, TiDB, and OceanBase rules+// that a backslash escapes the next character of a quoted string, that # starts a line comment, and+// that -- starts one only before whitespace or a control character.+func shapeKey(statement string, mysqlFamily bool) string {+ var b strings.Builder+ space := false+ // Whitespace is kept between words, where ? counts as a word, and in - - so that it never reads+ // as a comment marker.+ isWord := func(c byte) bool { return c == '?' || isWordByte(c) }+ write := func(c byte) {+ if space && b.Len() > 0 {+ if last := b.String()[b.Len()-1]; (isWord(last) && isWord(c)) || (last == '-' && c == '-') {+ b.WriteByte(' ')+ }+ }+ space = false+ b.WriteByte(c)}- totalAffectedRows += int64(changeSummary.InsertCount)- // Count affected rows by DDLs.- totalAffectedRows += changeSummary.ChangedResources.CountAffectedTableRows()+ for i := 0; i < len(statement); {+ c := statement[i]+ switch {+ case c == '\'':+ word := wordBefore(statement, i)+ if strings.EqualFold(word, "q") || strings.EqualFold(word, "nq") {+ return statement+ }+ // PostgreSQL E'...' and CockroachDB b'...' strings escape with backslashes too.+ end, closed := quotedEnd(statement, i, mysqlFamily || strings.EqualFold(word, "e") || strings.EqualFold(word, "b"))+ if !closed {+ return statement+ }+ i = end+ write('?')+ case c == '$' && !mysqlFamily && (i == 0 || !isWordByte(statement[i-1])) && isDollarQuoteStart(statement[i:]):+ return statement+ case c == '[':+ // A SQL Server bracketed identifier, where ]] continues it, or an array subscript, keeps its text.+ end := i + 1+ for end < len(statement) && (statement[end] != ']' || (end+1 < len(statement) && statement[end+1] == ']')) {+ if statement[end] == ']' {+ end+++ }+ end+++ }+ if end >= len(statement) || strings.ContainsAny(statement[i+1:end], `'"`) {+ return statement+ }+ write(c)+ b.WriteString(statement[i+1 : end+1])+ i = end + 1+ case c == '"' || c == '`':+ end, closed := quotedEnd(statement, i, mysqlFamily && c == '"')+ if !closed {+ return statement+ }+ write(c)+ b.WriteString(statement[i+1 : end])+ i = end+ case (strings.HasPrefix(statement[i:], "--") && (!mysqlFamily || i+2 == len(statement) || statement[i+2] <= ' ')) ||+ strings.HasPrefix(statement[i:], "/*") || (mysqlFamily && c == '#'):+ end := commentEnd(statement, i)+ if c == '/' && strings.Contains(statement[i+2:end], "/*") {+ return statement+ }+ if isKeptComment(statement[i:end]) {+ write(c)+ b.WriteString(statement[i+1 : end])+ } else {+ space = true+ }+ i = end+ case c >= '0' && c <= '9' && (i == 0 || !isWordByte(statement[i-1])):+ if end := numberEnd(statement, i, mysqlFamily); end >= 0 {+ i = end+ write('?')+ break+ }+ // A word that starts with digits but is not a number, such as [2024_orders], is an identifier.+ for i < len(statement) && isWordByte(statement[i]) {+ write(statement[i])+ i+++ }+ case c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v':+ space = true+ i+++ default:+ write(c)+ i+++ }+ }+ return b.String()+}++// wordBefore returns the word that ends at offset end of statement.+func wordBefore(statement string, end int) string {+ start := end+ for start > 0 && isWordByte(statement[start-1]) {+ start--+ }+ return statement[start:end]+}++// isDollarQuoteStart reports whether text starts with a dollar-quote delimiter, such as $$ or $tag$.+func isDollarQuoteStart(text string) bool {+ end := 1+ for end < len(text) && text[end] != '$' && isWordByte(text[end]) && (end > 1 || text[end] < '0' || text[end] > '9') {+ end+++ }+ return end < len(text) && text[end] == '$'+}++// isKeptComment reports whether a comment changes how a statement runs: an optimizer hint, such as+// /*+ ... */ or Oracle's --+ ..., or a MySQL, MariaDB, or TiDB executable comment, such as+// /*!80000 ... */, /*M!100100 ... */, or /*T![feature] ... */.+func isKeptComment(comment string) bool {+ for _, prefix := range []string{"/*+", "--+", "/*!", "/*M!", "/*T!"} {+ if strings.HasPrefix(comment, prefix) {+ return true+ }+ }+ return false+}++// commentEnd returns the offset just past the comment that starts at start: past the newline that+// ends a -- or # comment, which keeps the next line out of the comment's copied text, or past the */+// that closes a /* comment, or the end of the statement when it does not close.+func commentEnd(statement string, start int) int {+ if statement[start] != '/' {+ if end := strings.IndexByte(statement[start:], '\n'); end >= 0 {+ return start + end + 1+ }+ return len(statement)+ }+ if end := strings.Index(statement[start+2:], "*/"); end >= 0 {+ return start + 2 + end + 2+ }+ return len(statement)+}++// numberEnd returns the offset just past the number that starts at start, or -1 when the word there+// is not a number: a decimal, or a 0x hexadecimal or 0b binary integer. Outside mysqlFamily, where+// words such as 0X1F, 0o17, and 1_000 are identifiers, it also reads uppercase prefixes, PostgreSQL's+// 0o octal integers, and _ between digits.+func numberEnd(statement string, start int, mysqlFamily bool) int {+ isDigit := func(i int, digits string) bool {+ return i < len(statement) && (strings.IndexByte(digits, statement[i]) >= 0 || (!mysqlFamily && statement[i] == '_'))+ }+ prefixDigits := ""+ if start+1 < len(statement) && statement[start] == '0' {+ switch c := statement[start+1]; {+ case c == 'x' || (c == 'X' && !mysqlFamily):+ prefixDigits = "0123456789abcdefABCDEF"+ case c == 'b' || (c == 'B' && !mysqlFamily):+ prefixDigits = "01"+ case (c == 'o' || c == 'O') && !mysqlFamily:+ prefixDigits = "01234567"+ default:+ }+ }+ i := start+ if prefixDigits != "" {+ i += 2+ for isDigit(i, prefixDigits) {+ i+++ }+ if i == start+2 {+ return -1+ }+ } else {+ for isDigit(i, "0123456789.") {+ i+++ }+ if i < len(statement) && (statement[i] == 'e' || statement[i] == 'E') {+ exponent := i + 1+ if exponent < len(statement) && (statement[exponent] == '+' || statement[exponent] == '-') {+ exponent+++ }+ if isDigit(exponent, "0123456789") {+ i = exponent+ for isDigit(i, "0123456789") {+ i+++ }+ }+ }+ }+ if i < len(statement) && isWordByte(statement[i]) {+ return -1+ }+ return i+}++// quotedEnd returns the offset just past the quoted text that starts at start, where a doubled+// quote continues the text, and so does an escaped quote with backslashEscapes. It reports false+// when the text never closes.+func quotedEnd(statement string, start int, backslashEscapes bool) (int, bool) {+ quote := statement[start]+ for i := start + 1; i < len(statement); i++ {+ switch {+ case backslashEscapes && statement[i] == '\\':+ i+++ case statement[i] != quote:+ case i+1 < len(statement) && statement[i+1] == quote:+ i+++ default:+ return i + 1, true+ }+ }+ return len(statement), false+}- return totalAffectedRows+func isWordByte(c byte) bool {+ return c == '_' || c == '$' || c >= 0x80 || ('0' <= c && c <= '9') || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')}diff --git a/backend/runner/plancheck/statement_report_executor_test.go b/backend/runner/plancheck/statement_report_executor_test.goindex fac96fe4c5..c80ee2e877 100644--- a/backend/runner/plancheck/statement_report_executor_test.go+++ b/backend/runner/plancheck/statement_report_executor_test.go@@ -1,16 +1,308 @@package plancheckimport (+ "context"+ "fmt"+ "math""slices""testing"+ "github.com/pkg/errors""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model")+func TestCalculateAffectedRows(t *testing.T) {+ type estimate struct {+ statement string+ rows int64+ err error+ }+ repeat := func(count int, format string, rows int64) []estimate {+ var estimates []estimate+ for i := range count {+ estimates = append(estimates, estimate{fmt.Sprintf(format, i+1), rows, nil})+ }+ return estimates+ }+ for _, tc := range []struct {+ name string+ statements []estimate+ dmlCount int+ insertCount int+ wantRows int64+ wantWarning string+ wantExplains int+ }{+ {+ name: "every_dml_estimated_sums_exactly",+ statements: []estimate{{"u1", 10, nil}, {"u2", 17, nil}, {"u3", 34, nil}},+ dmlCount: 3,+ wantRows: 61,+ wantExplains: 3,+ },+ {+ name: "dml_without_text_counts_as_the_average_estimate",+ statements: []estimate{{"u1", 10, nil}, {"u2", 30, nil}},+ dmlCount: 5,+ wantRows: 100,+ wantWarning: "Affected rows could not be estimated for 3 of 5 DML statements.",+ wantExplains: 2,+ },+ {+ // The ten one-row updates share a shape, so the delete after them is still estimated.+ name: "samples_cover_every_shape",+ statements: append(repeat(10, "UPDATE t SET v = 'y' WHERE id = %d;", 1),+ estimate{"DELETE FROM big WHERE s_id <= 50;", 5000, nil}),+ dmlCount: 11,+ wantRows: 5010,+ wantExplains: 10,+ },+ {+ // Five statements of each shape are estimated: 2 * 15 + 100 * 5.+ name: "shape_average_counts_for_its_unsampled_statements",+ statements: append(repeat(15, "UPDATE t SET c = c + 1 WHERE id = %d;", 2),+ repeat(5, "DELETE FROM big WHERE s_id = %d;", 100)...),+ dmlCount: 20,+ wantRows: 530,+ wantExplains: 10,+ },+ {+ // The first ten shapes are estimated at 10 rows each; the other two count as that average.+ name: "shapes_beyond_the_samples_count_as_the_average_estimate",+ statements: append(repeat(10, "UPDATE t%d SET c = 1;", 10), estimate{"DELETE FROM a;", 1000, nil}, estimate{"DELETE FROM b;", 1000, nil}),+ dmlCount: 12,+ wantRows: 120,+ wantWarning: "Affected rows could not be estimated for 2 of 12 DML statements.",+ wantExplains: 10,+ },+ {+ name: "failed_sample_warns",+ statements: []estimate{{"u1", 3, nil}, {"d1", 0, errors.New("table t_new does not exist")}},+ dmlCount: 2,+ insertCount: 4,+ wantRows: 10,+ wantWarning: "Affected rows could not be estimated for 1 of 2 DML statements: table t_new does not exist",+ wantExplains: 2,+ },+ {+ // The shape's other statement is estimated, so the failed one counts as that estimate.+ name: "failed_sample_of_an_estimated_shape_warns",+ statements: []estimate{{"UPDATE t SET c = 1;", 0, errors.New("deadlock detected")}, {"UPDATE t SET c = 2;", 5, nil}},+ dmlCount: 2,+ wantRows: 10,+ wantWarning: "Affected rows could not be estimated for 1 of 2 DML statements: deadlock detected",+ wantExplains: 2,+ },+ {+ // One sampled shape fails and two shapes are never sampled; all three count as the average of the other nine.+ name: "failures_and_unsampled_shapes_warn_together",+ statements: append(append([]estimate{{"DELETE FROM missing;", 0, errors.New("table missing does not exist")}}, repeat(9, "UPDATE t%d SET c = 1;", 10)...), estimate{"DELETE FROM a;", 1000, nil}, estimate{"DELETE FROM b;", 1000, nil}),+ dmlCount: 12,+ wantRows: 120,+ wantWarning: "Affected rows could not be estimated for 3 of 12 DML statements: table missing does not exist",+ wantExplains: 10,+ },+ {+ name: "every_sample_failed",+ statements: []estimate{{"d1", 0, errors.New("syntax error")}},+ dmlCount: 1,+ insertCount: 4,+ wantRows: 4,+ wantWarning: "Affected rows could not be estimated for 1 of 1 DML statements: syntax error",+ wantExplains: 1,+ },+ {+ name: "dml_without_text",+ dmlCount: 2,+ wantWarning: "Affected rows could not be estimated for 2 of 2 DML statements.",+ },+ {+ name: "saturates_instead_of_overflowing",+ statements: []estimate{{"u1", math.MaxInt64/2 + 1, nil}, {"u2", math.MaxInt64/2 + 1, nil}},+ dmlCount: 3,+ wantRows: math.MaxInt64,+ wantWarning: "Affected rows could not be estimated for 1 of 3 DML statements.",+ wantExplains: 2,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ changeSummary := &parserbase.ChangeSummary{+ ChangedResources: model.NewChangedResources(nil /* dbMetadata */),+ DMLCount: tc.dmlCount,+ InsertCount: tc.insertCount,+ }+ estimates := map[string]estimate{}+ for _, e := range tc.statements {+ changeSummary.DMLStatements = append(changeSummary.DMLStatements, e.statement)+ estimates[e.statement] = e+ }+ explains := 0+ explain := func(_ context.Context, statement string) (int64, error) {+ explains+++ e := estimates[statement]+ return e.rows, e.err+ }++ rows, warning := calculateAffectedRows(context.Background(), storepb.Engine_POSTGRES, changeSummary, explain)+ require.Equal(t, tc.wantRows, rows)+ require.Equal(t, tc.wantWarning, warning)+ require.Equal(t, tc.wantExplains, explains)+ })+ }+}++func TestShapeKey(t *testing.T) {+ for _, tc := range []struct {+ name string+ statements []string+ mysqlFamily bool+ want string+ }{+ {+ name: "literals and spacing",+ statements: []string{+ "UPDATE t SET v = 'it''s' WHERE id = 1;",+ "UPDATE t SET v='x' WHERE id=20.5;",+ "UPDATE t\n\tSET v = 'C:\\' WHERE id = 3;",+ },+ want: "UPDATE t SET v=? WHERE id=?;",+ },+ {+ name: "MySQL family backslash escapes",+ statements: []string{+ "UPDATE t SET v = 'it\\'s' WHERE id = 1;",+ "UPDATE t SET v = 'C:\\\\' WHERE id = 2;",+ },+ mysqlFamily: true,+ want: "UPDATE t SET v=? WHERE id=?;",+ },+ {+ name: "PostgreSQL escape strings",+ statements: []string{+ "UPDATE t SET v = E'it\\'s' WHERE id = 1;",+ "UPDATE t SET v = E'x' WHERE id = 2;",+ },+ want: "UPDATE t SET v=E? WHERE id=?;",+ },+ {+ name: "CockroachDB byte strings",+ statements: []string{+ "UPDATE t SET v = b'it\\'s' WHERE id = 1;",+ "UPDATE t SET v = b'x' WHERE id = 2;",+ },+ want: "UPDATE t SET v=b? WHERE id=?;",+ },+ {+ name: "ordinary comments",+ statements: []string{+ "-- row 1\nUPDATE t SET v = 1 WHERE id = 1;",+ "/* row 2 */ UPDATE t SET v = 2 /* it's */ WHERE id = 2; -- done",+ "UPDATE/**/t SET v = 3 WHERE id = 3;",+ },+ want: "UPDATE t SET v=? WHERE id=?;",+ },+ {+ name: "MySQL family # comments",+ statements: []string{+ "# row 1\nUPDATE t SET v = 1 WHERE id = 1;",+ "UPDATE t SET v = 2 WHERE id = 2; # row 2",+ },+ mysqlFamily: true,+ want: "UPDATE t SET v=? WHERE id=?;",+ },+ {+ name: "MySQL family -- without whitespace is not a comment",+ statements: []string{+ "UPDATE t SET c = c--1 WHERE id = 1;",+ "UPDATE t SET c = c--2 WHERE id = 20;",+ },+ mysqlFamily: true,+ want: "UPDATE t SET c=c--? WHERE id=?;",+ },+ {+ name: "lists of literals",+ statements: []string{+ "DELETE FROM t WHERE id IN (1, 2, 3)",+ "DELETE FROM t WHERE id IN ('a', 'b', 'c')",+ },+ want: "DELETE FROM t WHERE id IN(?,?,?)",+ },+ {+ name: "numbers",+ statements: []string{"UPDATE t1 SET c2 = 0x1F WHERE t1.c3 > 1e3", "UPDATE t1 SET c2 = 7 WHERE t1.c3 > 1.5e-3", "UPDATE t1 SET c2 = 0B1010 WHERE t1.c3 > 0o17", "UPDATE t1 SET c2 = 0xFF_FF WHERE t1.c3 > 1_000.5"},+ want: "UPDATE t1 SET c2=? WHERE t1.c3>?",+ },+ {+ name: "leading-dot decimals",+ statements: []string{"UPDATE t SET v = .1 WHERE id > .5", "UPDATE t SET v = .10 WHERE id > .75e2"},+ want: "UPDATE t SET v=.? WHERE id>.?",+ },+ {+ name: "MySQL family numbers",+ statements: []string{"UPDATE t1 SET c2 = 0x1F WHERE t1.c3 > 1e3", "UPDATE t1 SET c2 = 0b101 WHERE t1.c3 > 7"},+ mysqlFamily: true,+ want: "UPDATE t1 SET c2=? WHERE t1.c3>?",+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ for _, statement := range tc.statements {+ require.Equal(t, tc.want, shapeKey(statement, tc.mysqlFamily), statement)+ }+ })+ }++ t.Run("statements that differ beyond literals keep separate shapes", func(t *testing.T) {+ for _, pair := range [][2]string{+ {`UPDATE "Orders" SET v = 1`, `UPDATE "orders" SET v = 1`},+ {`UPDATE "2024_orders" SET v = 1`, `UPDATE "2025_orders" SET v = 1`},+ {"UPDATE `Orders` SET v = 1", "UPDATE `orders` SET v = 1"},+ {"UPDATE Orders SET v = 1", "UPDATE orders SET v = 1"},+ {"UPDATE [2024_orders] SET v = 1", "UPDATE [2025_orders] SET v = 1"},+ {"UPDATE [1] SET v = 1", "UPDATE [2] SET v = 1"},+ {"UPDATE [t]]1] SET v = 1", "UPDATE [t]]2] SET v = 1"},+ {"UPDATE 2024_orders SET v = 1", "UPDATE 2025_orders SET v = 1"},+ {"DELETE FROM t WHERE id IN (1)", "DELETE FROM t WHERE id IN (1, 2)"},+ {"UPDATE t SET v = 1 /*!80000 WHERE id = 1 */", "UPDATE t SET v = 1"},+ {"UPDATE /*+ CARDINALITY(t 1) */ t SET v = 1", "UPDATE /*+ CARDINALITY(t 1000000) */ t SET v = 1"},+ {"UPDATE t SET v = 1 /*!80000 WHERE id = 1 */", "UPDATE t SET v = 1 /*!99999 WHERE id = 1 */"},+ {"UPDATE t SET v = 1 /*M!100100 WHERE id = 1 */", "UPDATE t SET v = 1 /*M!999999 WHERE id = 1 */"},+ {"UPDATE t SET v = 1 /*T![feature] WHERE id = 1 */", "UPDATE t SET v = 1"},+ {"UPDATE --+ INDEX(t i1)\nt SET v = 1", "UPDATE --+ INDEX(t i2)\nt SET v = 1"},+ {"UPDATE /*+ CARDINALITY(t 1000000) */ t SET v = 1", "UPDATE t SET v = 1"},+ {"UPDATE [t--x] SET v = 1 WHERE id = 1", "UPDATE [t--x] SET v = 1"},+ {"UPDATE t SET v = $$--$$ WHERE id = 1", "UPDATE t SET v = $$--$$"},+ {"UPDATE t SET v = 1 -- don't\nWHERE id = 1", "UPDATE t SET v = 1 -- don't\n"},+ {"UPDATE t SET v = v - -1 WHERE flag", "UPDATE t SET v = v --? WHERE flag"},+ {"UPDATE t SET v = 1 -- x\nWHERE id = 1", "UPDATE t SET v = 1 -- x WHERE id=?"},+ {"UPDATE t SET note = q'{it's}' WHERE id = 1 -- don't", "UPDATE t SET note = q'{it's}' WHERE id > 0 -- don't"},+ {"UPDATE t SET v = 1 /* a /* b */ it's */ WHERE id = 1 -- don't", "UPDATE t SET v = 1 /* a /* b */ it's */ WHERE id > 0 -- don't"},+ {"UPDATE t SET a = ARRAY[']'] WHERE id = 1 AND note <> 'a--b'", "UPDATE t SET a = ARRAY[']'] WHERE true AND note <> 'a--b'"},+ // The quote never closes, so each statement keeps its own shape.+ {"UPDATE t SET v = $$it's$$ WHERE id = 1", "UPDATE t SET v = $$it's$$"},+ } {+ require.NotEqual(t, shapeKey(pair[0], false), shapeKey(pair[1], false), pair[0])+ require.NotEqual(t, shapeKey(pair[0], true), shapeKey(pair[1], true), pair[0])+ }+ // On MySQL, a word such as 0X1F, 0o17, or 1_000 is an identifier.+ for _, pair := range [][2]string{{"UPDATE t SET v = 0X1F", "UPDATE t SET v = 0X20"}, {"UPDATE t SET v = 0o17", "UPDATE t SET v = 0o20"}, {"UPDATE t SET v = 1_000", "UPDATE t SET v = 1_001"}} {+ require.NotEqual(t, shapeKey(pair[0], true), shapeKey(pair[1], true), pair[0])+ }+ // -- starts a comment on PostgreSQL, but on MySQL only before whitespace.+ require.Equal(t, shapeKey("UPDATE t SET v = v--1 WHERE id = 1", false), shapeKey("UPDATE t SET v = v", false))+ require.NotEqual(t, shapeKey("UPDATE t SET v = v--1 WHERE id = 1", true), shapeKey("UPDATE t SET v = v", true))+ require.NotEqual(t, shapeKey("UPDATE t SET v = 'it\\'s' WHERE id = 1", true), shapeKey("UPDATE t SET v = 'it\\'s'", true))+ require.NotEqual(t, shapeKey(`UPDATE t SET v = "a\"b" WHERE id = 1`, true), shapeKey(`UPDATE t SET v = "a\"b"`, true))+ require.NotEqual(t, shapeKey("UPDATE t SET v = 1 # x\nWHERE id = 1", true), shapeKey("UPDATE t SET v = 1 # x WHERE id=?", true))+ require.NotEqual(t, shapeKey("UPDATE t SET note = $$it's$$ WHERE id = 1 -- don't", false), shapeKey("UPDATE t SET note = $$it's$$ WHERE id > 0 -- don't", false))+ require.NotEqual(t, shapeKey("UPDATE t SET note = $x$it's$x$ WHERE id = 1 -- don't", false), shapeKey("UPDATE t SET note = $x$it's$x$ WHERE id > 0 -- don't", false))+ })+}+// reportEngines derives the list from common.EngineSupportStatementReport so a// newly admitted engine is covered without editing this test.func reportEngines() []storepb.Engine {diff --git a/backend/runner/plancheck/test/mysql_statement_type_check.yaml b/backend/runner/plancheck/test/mysql_statement_type_check.yamldeleted file mode 100644index 7bc3638fee..0000000000--- a/backend/runner/plancheck/test/mysql_statement_type_check.yaml+++ /dev/null@@ -1,234 +0,0 @@-- statement: |-- UPDATE items,- (SELECT id FROM items- WHERE id IN- (SELECT id FROM items- WHERE retail / wholesale >= 1.3 AND quantity < 100))- AS discounted- SET items.retail = items.retail * 0.9- WHERE items.id = discounted.id;- changetype: 2- want: []-- statement: REPLACE INTO test2 VALUES (1, 'Old', '2014-08-20 18:47:00');- changetype: 2- want: []-- statement: |-- (SELECT 1 AS result UNION SELECT 2)- ORDER BY result DESC LIMIT 1;- changetype: 2- want: []-- statement: |-- INSERT INTO tbl_temp2 (fld_id)- SELECT tbl_temp1.fld_order_id- FROM tbl_temp1 WHERE tbl_temp1.fld_order_id > 100;- changetype: 1- want: []-- statement: IMPORT TABLE FROM '/tmp/mysql-files/*.sdi';- changetype: 1- want: []-- statement: TABLE a EXCEPT TABLE b;- changetype: 1- want: []-- statement: EXPLAIN SELECT c->>"$.name" AS name FROM jempn WHERE g > 2;- changetype: 1- want: []-- statement: |-- CREATE TABLE artists_and_works- SELECT artist.name, COUNT(work.artist_id) AS number_of_works- FROM artist LEFT JOIN work ON artist.id = work.artist_id- GROUP BY artist.id;- changetype: 1- want: []-- statement: |-- CREATE EVENT e- ON SCHEDULE- EVERY 5 SECOND- DO- BEGIN- DECLARE v INTEGER;- DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;-- SET v = 0;-- WHILE v < 5 DO- INSERT INTO t1 VALUES (0);- UPDATE t2 SET s1 = s1 + 1;- SET v = v + 1;- END WHILE;- END;- changetype: 1- want: []-- statement: |-- CREATE TABLE customers (- id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,- modified DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,- custinfo JSON,- INDEX zips( (CAST(custinfo->'$.zipcode' AS UNSIGNED ARRAY)) )- );- changetype: 1- want: []-- statement: |-- CREATE TABLE t1 (c1 INT);- ALTER TABLE t1 ADD COLUMN c2 INT GENERATED ALWAYS AS (c1 + 1) STORED;- changetype: 1- want: []-- statement: |-- CREATE TABLE t1 (- id INT,- year_col INT- )- PARTITION BY RANGE (year_col) (- PARTITION p0 VALUES LESS THAN (1991),- PARTITION p1 VALUES LESS THAN (1995),- PARTITION p2 VALUES LESS THAN (1999)- );- changetype: 1- want: []-- statement: ALTER SERVER s OPTIONS (USER 'sally');- changetype: 1- want: []-- statement: |-- ALTER LOGFILE GROUP lg_3- ADD UNDOFILE 'undo_10.dat'- INITIAL_SIZE=32M- ENGINE=NDBCLUSTER;- changetype: 1- want: []-- statement: ALTER FUNCTION hello COMMENT 'hello function';- changetype: 2- want: []-- statement: |-- CREATE DEFINER = 'admin'@'localhost' PROCEDURE account_count()- BEGIN- SELECT 'Number of accounts:', COUNT(*) FROM mysql.user;- END;- changetype: 2- want: []-- statement: ALTER PROCEDURE account_count COMMENT 'this account_count';- changetype: 1- want: []-- statement: |-- CREATE FUNCTION hello (s CHAR(20))- RETURNS CHAR(50) DETERMINISTIC- RETURN CONCAT('Hello, ',s,'!');-- SELECT hello('world');- changetype: 2- want: []-- statement: |-- delimiter //- CREATE PROCEDURE citycount (IN country CHAR(3), OUT cities INT)- BEGIN- SELECT COUNT(*) INTO cities FROM world.city- WHERE CountryCode = country;- END//- delimiter ;- changetype: 2- want: []-- statement: CALL citycount('JPN', @cities); -- cities in Japan- changetype: 2- want: []-- statement: ALTER EVENT no_such_event ON SCHEDULE EVERY '2:3' DAY_HOUR;- changetype: 2- want: []-- statement: |-- CREATE EVENT myevent- ON SCHEDULE- EVERY 6 HOUR- COMMENT 'A sample comment.'- DO- UPDATE myschema.mytable SET mycol = mycol + 1;- changetype: 2- want: []-- statement: ALTER DATABASE bbdev READ ONLY = DEFAULT;- changetype: 2- want: []-- statement: ALTER SCHEMA bbdev READ ONLY = DEFAULT;- changetype: 2- want: []-- statement: CREATE TABLE t1(a INT);- changetype: 1- want: []-- statement: DELETE FROM t1;- changetype: 1- want: []-- statement: SELECT 1;- changetype: 2- want: []-- statement: DROP TABLE t1;- changetype: 2- want: []-- statement: CREATE DATABASE bbdev;- changetype: 1- want:- - |-- {- "status": "ERROR",- "title": "Cannot create database",- "content": "The statement \"CREATE DATABASE bbdev;\" creates database",- "code": 404- }-- statement: DROP DATABASE bbdev;- changetype: 2- want:- - |-- {- "status": "ERROR",- "title": "Cannot drop database",- "content": "The statement \"DROP DATABASE bbdev;\" drops database",- "code": 403- }-- statement: RENAME TABLE t1 To t2;- changetype: 2- want: []-- statement: TRUNCATE TABLE t1;- changetype: 2- want: []-- statement: ALTER TABLE t1 ADD COLUMN col1 INT;- changetype: 2- want: []-- statement: DELETE FROM t1 WHERE id = 1;- changetype: 1- want: []-- statement: INSERT INTO t1(id) VALUE(1);- changetype: 1- want: []-- statement: UPDATE t1 SET name = 'zp' WHERE id = 1;- changetype: 1- want: []-- statement: |-- update t_ci_job_artifact_config c- inner join (- select- structure.id as structure_id,- structure.job_id as job_id- from- t_ci_job_stage_group_structure structure,- t_ci_stage_template_group stg- where- structure.stage_template_group_id = stg.id- and stg.template_type = 3- and stg.artifact_type in (4,5,6,7)- ) tmp on c.job_id = tmp.job_id and c.type in (4,5,6,7)- set c.structure_id = tmp.structure_id- where c.structure_id is null;- changetype: 1- want: []-- statement: |-- update t_ci_job_artifact_config c- inner join (- select- structure.id as structure_id,- structure.job_id as job_id- from- t_ci_job_stage_group_structure structure,- t_ci_stage_template_group stg- where- structure.stage_template_group_id = stg.id- and stg.template_type = 3- and stg.artifact_type in (4,5,6,7)- ) tmp on c.job_id = tmp.job_id and c.type in (4,5,6,7)- set c.structure_id = tmp.structure_id- where c.structure_id is null;- changetype: 2- want: []diff --git a/backend/runner/plancheck/test/test_affected_rows.yaml b/backend/runner/plancheck/test/test_affected_rows.yamldeleted file mode 100644index b61bd274f8..0000000000--- a/backend/runner/plancheck/test/test_affected_rows.yaml+++ /dev/null@@ -1,6 +0,0 @@-- statement: INSERT INTO t1 VALUES(1), (2), (3);- want: 3-- statement: ALTER TABLE t1 ADD COLUMN c2 INT;- want: 100-- statement: DROP TABLE t2;- want: 1000diff --git a/backend/store/model/changed_resources.go b/backend/store/model/changed_resources.goindex c61c7ef2f0..6407b36100 100644--- a/backend/store/model/changed_resources.go+++ b/backend/store/model/changed_resources.go@@ -2,6 +2,9 @@ package modelimport ("slices"+ "strings"++ metadatapb "github.com/bytebase/omni/metadata"storepb "github.com/bytebase/bytebase/backend/generated-go/store")@@ -47,15 +50,7 @@ func (r *ChangedResources) Build() *storepb.ChangedResources {d.Name = namefor _, schema := range d.Schemas {for _, table := range schema.Tables {- if r.dbMetadata == nil {- continue- }- schemaMetadata := r.dbMetadata.GetSchemaMetadata(schema.GetName())- if schemaMetadata == nil {- continue- }- tableMetadata := schemaMetadata.GetTable(table.GetName())- if tableMetadata != nil {+ if tableMetadata := r.getTableMetadata(name, schema.GetName(), table.GetName()); tableMetadata != nil {table.TableRows = tableMetadata.GetProto().GetRowCount()}}@@ -65,6 +60,25 @@ func (r *ChangedResources) Build() *storepb.ChangedResources {return changedResources}+// getTableMetadata returns the synced metadata of a changed table. The metadata only+// describes the reviewed database, so a table qualified with another database has none,+// even when the reviewed database has a table of the same name.+func (r *ChangedResources) getTableMetadata(database, schema, table string) *TableMetadata {+ if r.dbMetadata == nil {+ return nil+ }+ if name := r.dbMetadata.DatabaseName(); name != "" {+ if r.dbMetadata.GetIsObjectCaseSensitive() {+ if database != name {+ return nil+ }+ } else if !strings.EqualFold(database, name) {+ return nil+ }+ }+ return r.dbMetadata.GetSchemaMetadata(schema).GetTable(table)+}+func (d *ChangedDatabase) build() *storepb.ChangedResourceDatabase {changedResourceDatabase := &storepb.ChangedResourceDatabase{}for name, schema := range d.schemas {@@ -148,29 +162,22 @@ func (r *ChangedResources) GetDatabaseOnlyTargets() []string {return result}+// CountAffectedTableRows sums the synced row counts of the tables changed by DDL. Partitions+// have no row count of their own and resolve to their table, which is counted once.func (r *ChangedResources) CountAffectedTableRows() int64 {- if r.dbMetadata == nil {- return 0- }-+ counted := make(map[*metadatapb.TableMetadata]bool)var totalAffectedRows int64- for _, d := range r.databases {+ for databaseName, d := range r.databases {for schemaName, schema := range d.schemas {for tableName, table := range schema.tables {if !table.affectedTable {continue}- if r.dbMetadata == nil {- continue- }- schemaMeta := r.dbMetadata.GetSchemaMetadata(schemaName)- if schemaMeta == nil {- continue- }- tableMeta := schemaMeta.GetTable(tableName)- if tableMeta == nil {+ tableMeta := r.getTableMetadata(databaseName, schemaName, tableName)+ if tableMeta == nil || counted[tableMeta.GetProto()] {continue}+ counted[tableMeta.GetProto()] = truetotalAffectedRows += tableMeta.GetProto().GetRowCount()}}diff --git a/backend/store/model/changed_resources_test.go b/backend/store/model/changed_resources_test.gonew file mode 100644index 0000000000..84a9bc37ea--- /dev/null+++ b/backend/store/model/changed_resources_test.go@@ -0,0 +1,102 @@+package model++import (+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestCountAffectedTableRows(t *testing.T) {+ type change struct {+ database, schema, table string+ affected bool+ }+ for _, tc := range []struct {+ name string+ engine storepb.Engine+ changes []change+ want int64+ }{+ {+ name: "reviewed_database_table",+ engine: storepb.Engine_POSTGRES,+ changes: []change{{"db", "public", "t", true}},+ want: 1_000_000,+ },+ {+ name: "same_named_table_in_another_database",+ engine: storepb.Engine_POSTGRES,+ changes: []change{{"other_db", "public", "t", true}},+ want: 0,+ },+ {+ name: "table_changed_without_ddl",+ engine: storepb.Engine_POSTGRES,+ changes: []change{{"db", "public", "t", false}},+ want: 0,+ },+ {+ name: "partitions_count_their_table_once",+ // The metadata has no per-partition row counts.+ engine: storepb.Engine_POSTGRES,+ changes: []change{+ {"db", "public", "events", true},+ {"db", "public", "events_2020", true},+ {"db", "public", "events_2021", true},+ },+ want: 50_000_000,+ },+ {+ name: "case_insensitive_database_name",+ engine: storepb.Engine_MYSQL,+ changes: []change{{"DB", "public", "t", true}},+ want: 1_000_000,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ changedResources := NewChangedResources(newChangedResourcesTestMetadata(tc.engine))+ for _, c := range tc.changes {+ changedResources.AddTable(c.database, c.schema, &storepb.ChangedResourceTable{Name: c.table}, c.affected)+ }+ require.Equal(t, tc.want, changedResources.CountAffectedTableRows())+ })+ }+}++func TestBuildTableRows(t *testing.T) {+ changedResources := NewChangedResources(newChangedResourcesTestMetadata(storepb.Engine_POSTGRES))+ changedResources.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t"}, false)+ changedResources.AddTable("other_db", "public", &storepb.ChangedResourceTable{Name: "t"}, false)++ tableRows := map[string]int64{}+ for _, database := range changedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ tableRows[database.GetName()+"."+table.GetName()] = table.GetTableRows()+ }+ }+ }+ require.Equal(t, map[string]int64{"db.t": 1_000_000, "other_db.t": 0}, tableRows)+}++// newChangedResourcesTestMetadata is the reviewed database "db". MySQL metadata is built+// case-insensitive, as on a server with lower_case_table_names set.+func newChangedResourcesTestMetadata(engine storepb.Engine) *DatabaseMetadata {+ return NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {Name: "t", RowCount: 1_000_000},+ {+ Name: "events",+ RowCount: 50_000_000,+ Partitions: []*metadatapb.TablePartitionMetadata{{Name: "events_2020"}, {Name: "events_2021"}},+ },+ },+ }},+ }, nil, nil, engine, engine != storepb.Engine_MYSQL)+}diff --git a/backend/tests/gitops_test.go b/backend/tests/gitops_test.goindex 5718af8a3e..c77e215078 100644--- a/backend/tests/gitops_test.go+++ b/backend/tests/gitops_test.go@@ -98,6 +98,12 @@ func TestGitOpsCheck(t *testing.T) {Version: "002",Statement: []byte(`CREATE INDEX idx_users_email ON users(email);`),},+ {+ // Each file is checked against the database as it is, where users does not exist yet.+ Path: "migrations/003__normalize_emails.sql",+ Version: "003",+ Statement: []byte(`UPDATE users SET email = lower(email);`),+ },},}@@ -112,17 +118,32 @@ func TestGitOpsCheck(t *testing.T) {a.NoError(err)a.NotNil(checkResp)// The response contains results for each file-target combination- a.Len(checkResp.Msg.Results, 2) // 2 files x 1 target = 2 results+ a.Len(checkResp.Msg.Results, 3) // 3 files x 1 target = 3 results// Verify check results.targetCount := make(map[string]int)+ updateChecked := falsefor _, result := range checkResp.Msg.Results {targetCount[result.Target]++- // The check should complete successfully.- a.NotNil(result)+ var estimateWarnings []*v1pb.Advice+ for _, advice := range result.Advices {+ if advice.Title == "Affected rows estimate is incomplete" {+ estimateWarnings = append(estimateWarnings, advice)+ }+ }+ if result.File != "migrations/003__normalize_emails.sql" {+ a.Empty(estimateWarnings, result.File)+ continue+ }+ // The UPDATE cannot be explained before users exists, which the check reports instead of failing.+ updateChecked = true+ a.Len(estimateWarnings, 1)+ a.Equal(v1pb.Advice_WARNING, estimateWarnings[0].Status)+ a.Contains(estimateWarnings[0].Content, "Affected rows could not be estimated for 1 of 1 DML statements")}- // Should have 2 results for the single target (one for each file)- a.Equal(2, targetCount[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])+ a.True(updateChecked)+ // Should have 3 results for the single target (one for each file)+ a.Equal(3, targetCount[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])// Test 2: Check release against multiple targets (test and prod).checkRespMulti, err := ctl.releaseServiceClient.CheckRelease(ctx, connect.NewRequest(&v1pb.CheckReleaseRequest{@@ -135,17 +156,17 @@ func TestGitOpsCheck(t *testing.T) {}))a.NoError(err)a.NotNil(checkRespMulti)- // 2 files x 2 targets = 4 results- a.Len(checkRespMulti.Msg.Results, 4)+ // 3 files x 2 targets = 6 results+ a.Len(checkRespMulti.Msg.Results, 6)// Verify both targets were checked.checkedTargets := make(map[string]int)for _, result := range checkRespMulti.Msg.Results {checkedTargets[result.Target]++}- // Each target should have 2 results (one for each file)- a.Equal(2, checkedTargets[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])- a.Equal(2, checkedTargets[fmt.Sprintf("%s/databases/%s", prodInstance.Name, databaseName)])+ // Each target should have 3 results (one for each file)+ a.Equal(3, checkedTargets[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])+ a.Equal(3, checkedTargets[fmt.Sprintf("%s/databases/%s", prodInstance.Name, databaseName)])}func TestGitOpsCheckReleaseRiskLevel(t *testing.T) {diff --git a/backend/tests/sql_dml_table_scope_test.go b/backend/tests/sql_dml_table_scope_test.goindex b921ca72d9..06f1bc830b 100644--- a/backend/tests/sql_dml_table_scope_test.go+++ b/backend/tests/sql_dml_table_scope_test.go@@ -987,4 +987,50 @@ func TestSQLEditorTableScopedDMLEdgeCases(t *testing.T) {resp, qErr := runAs(token, fmt.Sprintf("INSERT INTO %s.public.t VALUES (1);", otherProjDB))assertDeniedOn(t, resp, qErr, fmt.Sprintf("databases/%s", otherProjDB))})++ // 18. A data-modifying CTE makes a SELECT a write: a SELECT-only user is denied, and a+ // table-scoped DML grant authorizes it per CTE target.+ t.Run("DataModifyingCTEAuthorizedAsDML", func(t *testing.T) {+ ra := require.New(t)+ email, token := newLimitedUser(t)+ setProjectBindings(t, readBinding(email))++ before := countRows(t, "public.t_src")+ resp, qErr := runAs(token, "WITH d AS (DELETE FROM public.t_src RETURNING id) SELECT count(*) FROM d;")+ assertDeniedOn(t, resp, qErr, "/tables/t_src")+ ra.Equal(before, countRows(t, "public.t_src"), "the denied statement must not delete rows")++ setProjectBindings(t, readBinding(email), tableScopedDML(email, "t_granted"))+ before = countRows(t, "public.t_granted")+ resp, qErr = runAs(token, "WITH i AS (INSERT INTO public.t_granted (id) VALUES (42) RETURNING id) SELECT id FROM i;")+ assertAllowed(t, resp, qErr)+ // The editor runs a write with Exec, so the rows the CTE returns never reach the response.+ ra.Equal([]string{"Affected Rows"}, resp.Results[0].ColumnNames)+ ra.Equal(before+1, countRows(t, "public.t_granted"))+ resp, qErr = runAs(token, "WITH i AS (INSERT INTO public.t_other (id) VALUES (42) RETURNING id) SELECT count(*) FROM i;")+ assertDeniedOn(t, resp, qErr, "/tables/t_other")+ })++ // 19. EXPLAIN ANALYZE executes the statement it explains, so explaining a write is authorized+ // like the write, per target table.+ t.Run("ExplainAnalyzeOfWriteAuthorizedAsDML", func(t *testing.T) {+ ra := require.New(t)+ email, token := newLimitedUser(t)+ setProjectBindings(t, readBinding(email), tableScopedDML(email, "t_granted"))++ before := countRows(t, "public.t_dst")+ for _, statement := range []string{+ "EXPLAIN ANALYZE MERGE INTO public.t_dst d USING public.t_src s ON d.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id);",+ "EXPLAIN ANALYZE WITH i AS (INSERT INTO public.t_dst (id) VALUES (1) RETURNING id) SELECT count(*) FROM i;",+ } {+ resp, qErr := runAs(token, statement)+ assertDeniedOn(t, resp, qErr, "/tables/t_dst")+ }+ ra.Equal(before, countRows(t, "public.t_dst"), "the denied statements must not write rows")++ before = countRows(t, "public.t_granted")+ resp, qErr := runAs(token, "EXPLAIN ANALYZE WITH i AS (INSERT INTO public.t_granted (id) VALUES (43) RETURNING id) SELECT count(*) FROM i;")+ assertAllowed(t, resp, qErr)+ ra.Equal(before+1, countRows(t, "public.t_granted"))+ })}