Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(plancheck): estimate affected rows correctly across engines (#21442)

Danny Xue22be6f5e2d6
diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.go
index f28ba63bd9..125975f2ae 100644
--- a/backend/api/v1/release_service_check.go
+++ b/backend/api/v1/release_service_check.go
@@ -447,18 +447,26 @@ loop:
// Get SQL summary report for the statement and target database.
// Including affected rows.
- summaryReport, err := plancheck.GetSQLSummaryReport(ctx, s.store, s.sheetManager, s.dbFactory, database, statement)
+ summaryReport, estimateWarning, err := plancheck.GetSQLSummaryReport(ctx, s.store, s.sheetManager, s.dbFactory, database, statement)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get SQL summary report"))
}
if summaryReport != nil {
checkResult.AffectedRows = summaryReport.AffectedRows
checkResult.RiskLevel = getRiskLevelFromStatementTypes(summaryReport.StatementTypes)
- resp.AffectedRows += summaryReport.AffectedRows
+ resp.AffectedRows = common.AddRows(resp.AffectedRows, summaryReport.AffectedRows)
if checkResult.RiskLevel > resp.RiskLevel {
resp.RiskLevel = checkResult.RiskLevel
}
}
+ if estimateWarning != "" {
+ checkResult.Advices = append(checkResult.Advices, &v1pb.Advice{
+ Status: v1pb.Advice_WARNING,
+ Code: code.StatementExplainQueryFailed.Int32(),
+ Title: plancheck.AffectedRowsEstimateIncompleteTitle,
+ Content: estimateWarning,
+ })
+ }
if common.EngineSupportSQLReview(engine) {
adviceStatus, sqlReviewAdvices, err := s.runSQLReviewCheckForFile(ctx, project, originMetadata, finalMetadata, instance, database, statement)
if err != nil {
diff --git a/backend/common/engine.go b/backend/common/engine.go
index 7ade4471cb..09870374f4 100644
--- a/backend/common/engine.go
+++ b/backend/common/engine.go
@@ -246,7 +246,8 @@ func EngineSupportStatementReport(e storepb.Engine) bool {
storepb.Engine_ORACLE,
storepb.Engine_MSSQL,
storepb.Engine_MARIADB,
- storepb.Engine_REDSHIFT:
+ storepb.Engine_REDSHIFT,
+ storepb.Engine_COCKROACHDB:
return true
case
storepb.Engine_ENGINE_UNSPECIFIED,
@@ -259,7 +260,6 @@ func EngineSupportStatementReport(e storepb.Engine) bool {
storepb.Engine_BIGQUERY,
storepb.Engine_STARROCKS,
storepb.Engine_HIVE,
- storepb.Engine_COCKROACHDB,
storepb.Engine_DORIS,
storepb.Engine_DYNAMODB,
storepb.Engine_ELASTICSEARCH,
diff --git a/backend/common/util.go b/backend/common/util.go
index 85a55f6a31..003fdc86a8 100644
--- a/backend/common/util.go
+++ b/backend/common/util.go
@@ -5,6 +5,7 @@ import (
"crypto/rand"
"encoding/base64"
"fmt"
+ "math"
"math/big"
"net/url"
"reflect"
@@ -34,6 +35,23 @@ const (
MaximumLintExplainSize = 10
)
+// RoundRows rounds a planner's row estimate to int64, saturating at math.MaxInt64 instead of
+// overflowing to a negative count.
+func RoundRows(rows float64) int64 {
+ if rows >= math.MaxInt64 {
+ return math.MaxInt64
+ }
+ return int64(math.Round(rows))
+}
+
+// AddRows adds row counts, saturating at math.MaxInt64 instead of wrapping negative.
+func AddRows(a, b int64) int64 {
+ if b > 0 && a > math.MaxInt64-b {
+ return math.MaxInt64
+ }
+ return a + b
+}
+
var letters = []rune("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
// ProtojsonMarshaler is a global protojson marshaler with DiscardUnknown set to true.
diff --git a/backend/common/util_test.go b/backend/common/util_test.go
index 00ee1d1334..2366732da3 100644
--- a/backend/common/util_test.go
+++ b/backend/common/util_test.go
@@ -2,6 +2,7 @@
package common
import (
+ "math"
"testing"
"unicode/utf8"
@@ -449,3 +450,16 @@ func TestSanitizeUTF8MessageNameFields(t *testing.T) {
"valid UTF-8 must pass through unchanged")
require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))
}
+
+func TestRoundRows(t *testing.T) {
+ require.Equal(t, int64(3), RoundRows(2.5))
+ require.Equal(t, int64(1000), RoundRows(999.6))
+ require.Equal(t, int64(math.MaxInt64), RoundRows(math.MaxInt64))
+ require.Equal(t, int64(math.MaxInt64), RoundRows(1e30))
+}
+
+func TestAddRows(t *testing.T) {
+ require.Equal(t, int64(5), AddRows(2, 3))
+ require.Equal(t, int64(math.MaxInt64), AddRows(math.MaxInt64, 1))
+ require.Equal(t, int64(math.MaxInt64), AddRows(math.MaxInt64-1, math.MaxInt64-1))
+}
diff --git a/backend/plugin/advisor/contains_ddl_test.go b/backend/plugin/advisor/contains_ddl_test.go
index caf74dc1b5..d639e39f9b 100644
--- a/backend/plugin/advisor/contains_ddl_test.go
+++ b/backend/plugin/advisor/contains_ddl_test.go
@@ -29,6 +29,12 @@ func TestContainsDDL(t *testing.T) {
sql: "INSERT INTO t VALUES (1); UPDATE t SET a = 1; DELETE FROM t;",
wantDDL: false,
},
+ {
+ name: "PostgreSQL: MERGE is DML",
+ engine: storepb.Engine_POSTGRES,
+ sql: "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE; UPDATE t SET a = 1;",
+ wantDDL: false,
+ },
{
name: "PostgreSQL: DDL only",
engine: storepb.Engine_POSTGRES,
diff --git a/backend/plugin/advisor/mysql/common.go b/backend/plugin/advisor/mysql/common.go
deleted file mode 100644
index 935302540f..0000000000
--- a/backend/plugin/advisor/mysql/common.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package mysql
-
-import "github.com/pkg/errors"
-
-func getColumnIndex(columns []string, name string) (int, error) {
- for i, columnName := range columns {
- if columnName == name {
- return i, nil
- }
- }
- return -1, errors.Errorf("failed to find column %q", name)
-}
diff --git a/backend/plugin/advisor/mysql/mysql_rules_test.go b/backend/plugin/advisor/mysql/mysql_rules_test.go
index 1e2cf18f05..a7428fd6b3 100644
--- a/backend/plugin/advisor/mysql/mysql_rules_test.go
+++ b/backend/plugin/advisor/mysql/mysql_rules_test.go
@@ -4,14 +4,16 @@ import (
"context"
"database/sql"
"database/sql/driver"
- "errors"
"io"
"strings"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
"github.com/stretchr/testify/require"
+ "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/component/sheet"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
@@ -19,20 +21,24 @@ import (
"github.com/bytebase/bytebase/backend/store/model"
)
-var testMySQLAdvisorExplainRows [][]driver.Value
+var (
+ // testMySQLAdvisorExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries.
+ testMySQLAdvisorExplainJSON string
-// testMySQLAdvisorExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries; when
-// empty, the JSON query returns no rows and the advisor falls back to the tabular rows.
-var testMySQLAdvisorExplainJSON string
+ // testMySQLAdvisorExplainErr, when set, fails every query, standing in for a
+ // statement the server refuses to EXPLAIN.
+ testMySQLAdvisorExplainErr error
-// testMySQLAdvisorExplainErr, when set, fails every query, standing in for a
-// statement the server refuses to EXPLAIN.
-var testMySQLAdvisorExplainErr error
+ // testMySQLAdvisorSetErr, when set, fails every SET statement.
+ testMySQLAdvisorSetErr error
-// testMySQLAdvisorQueries records every statement the fake driver is asked to
-// run, so a rule that sent raw DML instead of EXPLAIN is caught rather than
-// silently succeeding.
-var testMySQLAdvisorQueries []string
+ // testMySQLAdvisorQueries records every statement the fake driver is asked to
+ // run, so a rule that sent raw DML instead of EXPLAIN is caught rather than
+ // silently succeeding. testMySQLAdvisorQueryConns records the connection that ran each.
+ testMySQLAdvisorQueries []string
+ testMySQLAdvisorQueryConns []int
+ testMySQLAdvisorConnections int
+)
func init() {
sql.Register("test_mysql_advisor_explain", testMySQLAdvisorExplainDriver{})
@@ -41,39 +47,48 @@ func init() {
type testMySQLAdvisorExplainDriver struct{}
func (testMySQLAdvisorExplainDriver) Open(string) (driver.Conn, error) {
- return testMySQLAdvisorExplainConn{}, nil
+ testMySQLAdvisorConnections++
+ return &testMySQLAdvisorExplainConn{id: testMySQLAdvisorConnections}, nil
}
-type testMySQLAdvisorExplainConn struct{}
+type testMySQLAdvisorExplainConn struct {
+ id int
+}
-func (testMySQLAdvisorExplainConn) Prepare(string) (driver.Stmt, error) {
+func (*testMySQLAdvisorExplainConn) Prepare(string) (driver.Stmt, error) {
return nil, driver.ErrSkip
}
-func (testMySQLAdvisorExplainConn) Close() error {
+func (*testMySQLAdvisorExplainConn) Close() error {
return nil
}
-func (testMySQLAdvisorExplainConn) Begin() (driver.Tx, error) {
+func (*testMySQLAdvisorExplainConn) Begin() (driver.Tx, error) {
return testMySQLAdvisorExplainTx{}, nil
}
-func (testMySQLAdvisorExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+func (c *testMySQLAdvisorExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {
+ testMySQLAdvisorQueries = append(testMySQLAdvisorQueries, query)
+ testMySQLAdvisorQueryConns = append(testMySQLAdvisorQueryConns, c.id)
+ if !strings.HasPrefix(query, "SET ") {
+ return nil, errors.Errorf("unexpected exec %q", query)
+ }
+ if testMySQLAdvisorSetErr != nil {
+ return nil, testMySQLAdvisorSetErr
+ }
+ return driver.ResultNoRows, nil
+}
+
+func (c *testMySQLAdvisorExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
testMySQLAdvisorQueries = append(testMySQLAdvisorQueries, query)
+ testMySQLAdvisorQueryConns = append(testMySQLAdvisorQueryConns, c.id)
if testMySQLAdvisorExplainErr != nil {
return nil, testMySQLAdvisorExplainErr
}
- if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON") {
- var rows [][]driver.Value
- if testMySQLAdvisorExplainJSON != "" {
- rows = [][]driver.Value{{testMySQLAdvisorExplainJSON}}
- }
- return &testMySQLAdvisorExplainResultRows{columns: []string{"EXPLAIN"}, rows: rows}, nil
+ if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON ") {
+ return &testMySQLAdvisorExplainResultRows{columns: []string{"EXPLAIN"}, rows: [][]driver.Value{{testMySQLAdvisorExplainJSON}}}, nil
}
- return &testMySQLAdvisorExplainResultRows{
- columns: []string{"id", "select_type", "table", "type", "rows", "filtered"},
- rows: testMySQLAdvisorExplainRows,
- }, nil
+ return &testMySQLAdvisorExplainResultRows{columns: []string{"id", "select_type", "table", "type", "rows", "filtered"}}, nil
}
type testMySQLAdvisorExplainTx struct{}
@@ -187,95 +202,235 @@ func TestMySQLRules(t *testing.T) {
}
}
-func TestMySQLAffectedRowLimitAdvisorUsesJSONPlanTargetNode(t *testing.T) {
- // The tabular EXPLAIN's first row is a 100232-row driving-table scan; the JSON
- // plan flags the update target whose cumulative estimate is 1144 (BYT-9858).
- testMySQLAdvisorExplainJSON = `{"query_block":{"select_id":1,"nested_loop":[
- {"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":100232,"rows_produced_per_join":3006,"filtered":"3.00"}},
- {"table":{"update":true,"table_name":"o","access_type":"ref","rows_examined_per_scan":3,"rows_produced_per_join":1144,"filtered":"10.00"}}
- ]}}`
- testMySQLAdvisorExplainRows = [][]driver.Value{
- {int64(1), "SIMPLE", "t", "ALL", int64(100232), "3.00"},
- {int64(1), "UPDATE", "o", "ref", int64(3), "10.00"},
- }
- defer func() { testMySQLAdvisorExplainJSON = "" }()
-
+// openTestMySQLAdvisorDB resets the fake server and opens a pool that keeps no idle
+// connections, so statements that do not share a pinned connection land on different ones.
+func openTestMySQLAdvisorDB(t *testing.T, plan string) *sql.DB {
+ t.Helper()
+ testMySQLAdvisorExplainJSON = plan
+ testMySQLAdvisorExplainErr = nil
+ testMySQLAdvisorSetErr = nil
+ testMySQLAdvisorQueries = nil
+ testMySQLAdvisorQueryConns = nil
db, err := sql.Open("test_mysql_advisor_explain", "")
require.NoError(t, err)
- defer db.Close()
+ db.SetMaxIdleConns(0)
+ t.Cleanup(func() { _ = db.Close() })
+ return db
+}
- sm := sheet.NewManager()
- adviceList, err := advisor.SQLReviewCheck(context.Background(), sm, "UPDATE o SET c = 1 WHERE c IS NULL AND EXISTS (SELECT 1 FROM t WHERE t.k = o.k);", []*storepb.SQLReviewRule{
+func checkTestRowLimitRule(t *testing.T, db *sql.DB, engine storepb.Engine, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, statement string) []*storepb.Advice {
+ t.Helper()
+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{
{
- Type: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,
- Level: storepb.SQLReviewRule_WARNING,
- Engine: storepb.Engine_MYSQL,
- Payload: &storepb.SQLReviewRule_NumberPayload{
- NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{
- Number: 2000,
- },
- },
+ Type: ruleType,
+ Level: level,
+ Engine: engine,
+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: 5}},
},
}, advisor.Context{
- DBType: storepb.Engine_MYSQL,
+ DBType: engine,
Driver: db,
NoAppendBuiltin: true,
})
require.NoError(t, err)
- // 1144 is below the 2000 limit; the legacy first-"rows" heuristic would have
- // reported 100232 and produced advice.
- require.Empty(t, adviceList)
+ return adviceList
}
-func TestMySQLRowLimitAdvisorsCapExplainEstimateByLimit(t *testing.T) {
- testMySQLAdvisorExplainJSON = ""
- testMySQLAdvisorExplainRows = [][]driver.Value{
- {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},
- }
- db, err := sql.Open("test_mysql_advisor_explain", "")
- require.NoError(t, err)
- defer db.Close()
+const (
+ // The flagged UPDATE target is scanned first; the join filter on s comes after it.
+ testMySQLUpdateTargetFirstPlan = `{"query_block":{"select_id":1,"nested_loop":[
+ {"table":{"update":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00"}},
+ {"table":{"table_name":"s","access_type":"eq_ref","rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00"}}
+ ]}}`
+ // MariaDB does not flag a multi-table DELETE target.
+ testMariaDBDeleteJoinPlan = `{"query_block":{"select_id":1,"nested_loop":[
+ {"table":{"table_name":"s","access_type":"ALL","loops":1,"rows":100,"filtered":10}},
+ {"table":{"table_name":"big","access_type":"ref","loops":10,"rows":100,"filtered":100}}
+ ]}}`
+ // The plan of the SELECT that a single-table UPDATE of td is explained as.
+ testMySQLSingleTableUpdatePlan = `{"query_block":{"select_id":1,"table":{"table_name":"td","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00"}}}`
+ testMySQLInsertSelectPlan = `{"query_block":{"select_id":1,
+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},
+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}
+ }}`
+ testMySQLInsertTablePlan = `{"query_block":{"select_id":1,
+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},
+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00"}}
+ }}`
+)
- sm := sheet.NewManager()
+func TestMySQLRowLimitAdvisorsEstimateFromJSONPlan(t *testing.T) {
for _, tc := range []struct {
name string
- statement string
+ engine storepb.Engine
ruleType storepb.SQLReviewRule_Type
+ plan string
+ setErr error
+ statement string
+ // explained is the statement the rule explains, when it is not the statement itself.
+ explained string
+ // wantContent is empty when the statement stays within the limit of 5.
+ wantContent string
+ wantCode code.Code
+ // fallback is set when the JSON plan has no estimate to read, so the rule also runs the tabular EXPLAIN.
+ fallback bool
}{
{
- name: "affected row limit",
- statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 3;",
+ name: "update bounded by the join after the target",
+ engine: storepb.Engine_MYSQL,
+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,
+ plan: testMySQLUpdateTargetFirstPlan,
+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",
+ wantContent: `"UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;" affected 1133 rows (estimated). The count exceeds 5.`,
+ wantCode: code.StatementAffectedRowExceedsLimit,
+ },
+ {
+ name: "MariaDB multi-table delete without explain_json_format_version",
+ engine: storepb.Engine_MARIADB,
+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,
+ plan: testMariaDBDeleteJoinPlan,
+ setErr: &mysql.MySQLError{Number: 1193, Message: "Unknown system variable 'explain_json_format_version'"},
+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ wantContent: `"DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;" affected 1000 rows (estimated). The count exceeds 5.`,
+ wantCode: code.StatementAffectedRowExceedsLimit,
+ },
+ {
+ name: "update capped by LIMIT",
+ engine: storepb.Engine_MYSQL,
ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,
+ plan: testMySQLSingleTableUpdatePlan,
+ statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 3;",
+ explained: "SELECT 1 FROM td WHERE c = 0 LIMIT 3",
+ },
+ {
+ name: "target the plan does not name",
+ engine: storepb.Engine_MYSQL,
+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,
+ plan: `{"query_block":{"select_id":1,"table":{"table_name":"other","access_type":"ALL","rows_examined_per_scan":1000,"filtered":"100.00"}}}`,
+ statement: "UPDATE td SET c = 1;",
+ explained: "SELECT 1 FROM td",
+ wantContent: `"UPDATE td SET c = 1;" affected 1000 rows (estimated). The count exceeds 5.`,
+ wantCode: code.StatementAffectedRowExceedsLimit,
},
{
- name: "insert row limit",
- statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 3;",
+ name: "plan without an estimate falls back to the tabular plan",
+ engine: storepb.Engine_MYSQL,
+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,
+ plan: `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`,
+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;",
+ wantContent: `failed to get row count for "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;": table "t2" in the plan has no row estimate`,
+ wantCode: code.Internal,
+ fallback: true,
+ },
+ {
+ name: "insert select reads the source plan",
+ engine: storepb.Engine_MYSQL,
+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,
+ plan: testMySQLInsertSelectPlan,
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ wantContent: `"INSERT INTO t2 SELECT * FROM t WHERE d = 3;" inserts 100 rows. The count exceeds 5.`,
+ wantCode: code.InsertTooManyRows,
+ },
+ {
+ name: "insert table",
+ engine: storepb.Engine_MYSQL,
+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,
+ plan: testMySQLInsertTablePlan,
+ statement: "INSERT INTO t2 TABLE t;",
+ wantContent: `"INSERT INTO t2 TABLE t;" inserts 1000 rows. The count exceeds 5.`,
+ wantCode: code.InsertTooManyRows,
+ },
+ {
+ name: "insert select capped by LIMIT",
+ engine: storepb.Engine_MYSQL,
ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,
+ plan: testMySQLInsertSelectPlan,
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 3;",
},
} {
t.Run(tc.name, func(t *testing.T) {
- adviceList, err := advisor.SQLReviewCheck(context.Background(), sm, tc.statement, []*storepb.SQLReviewRule{
- {
- Type: tc.ruleType,
- Level: storepb.SQLReviewRule_WARNING,
- Engine: storepb.Engine_MYSQL,
- Payload: &storepb.SQLReviewRule_NumberPayload{
- NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{
- Number: 5,
- },
- },
- },
- }, advisor.Context{
- DBType: storepb.Engine_MYSQL,
- Driver: db,
- NoAppendBuiltin: true,
- })
- require.NoError(t, err)
- require.Empty(t, adviceList)
+ db := openTestMySQLAdvisorDB(t, tc.plan)
+ testMySQLAdvisorSetErr = tc.setErr
+
+ adviceList := checkTestRowLimitRule(t, db, tc.engine, tc.ruleType, storepb.SQLReviewRule_WARNING, tc.statement)
+ if tc.wantContent == "" {
+ require.Empty(t, adviceList)
+ } else {
+ require.Len(t, adviceList, 1)
+ require.Equal(t, tc.wantCode.Int32(), adviceList[0].Code)
+ require.Equal(t, tc.wantContent, adviceList[0].Content)
+ }
+
+ explained := tc.explained
+ if explained == "" {
+ explained = tc.statement
+ }
+ // The rule sets the JSON format version on the connection that runs the JSON EXPLAIN and
+ // resets it where the server has the variable.
+ wantQueries := []string{"SET SESSION explain_json_format_version = 1", "EXPLAIN FORMAT=JSON " + explained}
+ if tc.setErr == nil {
+ wantQueries = append(wantQueries, "SET SESSION explain_json_format_version = DEFAULT")
+ }
+ jsonQueries := len(wantQueries)
+ if tc.fallback {
+ wantQueries = append(wantQueries, "EXPLAIN FORMAT=TRADITIONAL "+explained)
+ }
+ require.Equal(t, wantQueries, testMySQLAdvisorQueries)
+ for _, conn := range testMySQLAdvisorQueryConns[:jsonQueries] {
+ require.Equal(t, testMySQLAdvisorQueryConns[0], conn)
+ }
})
}
}
+func TestMySQLRowLimitAdvisorsWarnAboutStatementsBeyondExplainLimit(t *testing.T) {
+ t.Run("affected row limit", func(t *testing.T) {
+ db := openTestMySQLAdvisorDB(t, testMySQLSingleTableUpdatePlan)
+ statement := strings.Repeat("UPDATE td SET c = 1;\n", common.MaximumLintExplainSize+2)
+
+ adviceList := checkTestRowLimitRule(t, db, storepb.Engine_MYSQL, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, statement)
+ require.Len(t, adviceList, common.MaximumLintExplainSize+1)
+ for _, advice := range adviceList[:common.MaximumLintExplainSize] {
+ require.Equal(t, storepb.Advice_ERROR, advice.Status)
+ require.Contains(t, advice.Content, "affected 1000 rows")
+ }
+ warning := adviceList[common.MaximumLintExplainSize]
+ require.Equal(t, storepb.Advice_WARNING, warning.Status)
+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), warning.Code)
+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT.String(), warning.Title)
+ require.Equal(t, "Only the first 10 statements were estimated; 2 more were not checked against the row limit.", warning.Content)
+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())
+ require.Equal(t, common.MaximumLintExplainSize, countTestExplainQueries())
+ })
+
+ t.Run("insert row limit keeps counting VALUES rows", func(t *testing.T) {
+ db := openTestMySQLAdvisorDB(t, testMySQLInsertSelectPlan)
+ statement := strings.Repeat("INSERT INTO t2 SELECT * FROM t WHERE d = 3;\n", common.MaximumLintExplainSize+1) +
+ "INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);"
+
+ adviceList := checkTestRowLimitRule(t, db, storepb.Engine_MYSQL, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, statement)
+ require.Len(t, adviceList, common.MaximumLintExplainSize+2)
+ require.Equal(t, `"INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);" inserts 6 rows. The count exceeds 5.`, adviceList[common.MaximumLintExplainSize].Content)
+ warning := adviceList[common.MaximumLintExplainSize+1]
+ require.Equal(t, storepb.Advice_WARNING, warning.Status)
+ require.Equal(t, code.InsertTooManyRows.Int32(), warning.Code)
+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT.String(), warning.Title)
+ require.Equal(t, "Only the first 10 statements were estimated; 1 more were not checked against the row limit.", warning.Content)
+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())
+ require.Equal(t, common.MaximumLintExplainSize, countTestExplainQueries())
+ })
+}
+
+func countTestExplainQueries() int {
+ count := 0
+ for _, query := range testMySQLAdvisorQueries {
+ if strings.HasPrefix(query, "EXPLAIN ") {
+ count++
+ }
+ }
+ return count
+}
+
func TestMariaDBPriorBackupCheckAdvisor(t *testing.T) {
sm := sheet.NewManager()
diff --git a/backend/plugin/advisor/mysql/rule_insert_row_limit.go b/backend/plugin/advisor/mysql/rule_insert_row_limit.go
index 9ae5f1c020..8ad5558b0b 100644
--- a/backend/plugin/advisor/mysql/rule_insert_row_limit.go
+++ b/backend/plugin/advisor/mysql/rule_insert_row_limit.go
@@ -3,7 +3,6 @@ package mysql
import (
"context"
"fmt"
- "strconv"
"strings"
"github.com/bytebase/omni/mysql/ast"
@@ -13,6 +12,7 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
+ mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"
mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
)
@@ -43,8 +43,9 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contex
maxRow := int(numberPayload.Number)
driver := checkCtx.Driver
+ title := checkCtx.Rule.Type.String()
var advice []*storepb.Advice
- explainCount := 0
+ var explains advisor.ExplainBudget
for _, stmt := range checkCtx.ParsedStatements {
if stmt.AST == nil {
@@ -61,45 +62,47 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contex
baseLine := stmt.BaseLine()
text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"
- line := baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line)
+ position := common.ConvertANTLRLineToPosition(baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line))
- // INSERT ... SELECT: use EXPLAIN to count rows.
- if ins.Select != nil {
+ // INSERT ... SELECT and INSERT ... TABLE: use EXPLAIN to count rows.
+ if ins.Select != nil || ins.TableSource != nil {
if driver == nil {
continue
}
- explainCount++
- res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN %s", text))
+ if !explains.Spend(position) {
+ continue
+ }
+
+ plan, err := mysqldriver.ExplainJSON(ctx, driver, text)
if err != nil {
advice = append(advice, &storepb.Advice{
Status: level,
Code: code.InsertTooManyRows.Int32(),
- Title: checkCtx.Rule.Type.String(),
+ Title: title,
Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(line),
+ StartPosition: position,
+ })
+ continue
+ }
+ rowCount, err := mysqldriver.EstimateAffectedRows(ctx, driver, ins, text, plan)
+ if err != nil {
+ advice = append(advice, &storepb.Advice{
+ Status: level,
+ Code: code.Internal.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),
+ StartPosition: position,
})
- } else {
- rowCount, err := getInsertRows(res)
- if err != nil {
- advice = append(advice, &storepb.Advice{
- Status: level,
- Code: code.Internal.Int32(),
- Title: checkCtx.Rule.Type.String(),
- Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(line),
- })
- } else if rowCount = capRowsByLimit(rowCount, ins.Select.Limit); rowCount > int64(maxRow) {
- advice = append(advice, &storepb.Advice{
- Status: level,
- Code: code.InsertTooManyRows.Int32(),
- Title: checkCtx.Rule.Type.String(),
- Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, rowCount, maxRow),
- StartPosition: common.ConvertANTLRLineToPosition(line),
- })
- }
+ continue
}
- if explainCount >= common.MaximumLintExplainSize {
- break
+ if rowCount > int64(maxRow) {
+ advice = append(advice, &storepb.Advice{
+ Status: level,
+ Code: code.InsertTooManyRows.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, rowCount, maxRow),
+ StartPosition: position,
+ })
}
continue
}
@@ -109,14 +112,14 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contex
advice = append(advice, &storepb.Advice{
Status: level,
Code: code.InsertTooManyRows.Int32(),
- Title: checkCtx.Rule.Type.String(),
+ Title: title,
Content: fmt.Sprintf("\"%s\" inserts %d rows. The count exceeds %d.", text, len(ins.Values), maxRow),
- StartPosition: common.ConvertANTLRLineToPosition(line),
+ StartPosition: position,
})
}
}
- return advice, nil
+ return explains.AppendSkippedAdvice(advice, title, code.InsertTooManyRows), nil
}
func contentStartIndex(text string) int {
@@ -127,70 +130,3 @@ func contentStartIndex(text string) int {
}
return 0
}
-
-func capRowsByLimit(rowCount int64, limit *ast.Limit) int64 {
- limitCount, ok := limitRowCount(limit)
- if !ok || limitCount >= rowCount {
- return rowCount
- }
- return limitCount
-}
-
-func limitRowCount(limit *ast.Limit) (int64, bool) {
- if limit == nil || limit.Count == nil {
- return 0, false
- }
- lit, ok := limit.Count.(*ast.IntLit)
- if !ok || lit.Value < 0 {
- return 0, false
- }
- return lit.Value, true
-}
-
-func getInsertRows(res []any) (int64, error) {
- // the res struct is []any{columnName, columnTable, rowDataList}
- if len(res) != 3 {
- return 0, errors.Errorf("expected 3 but got %d", len(res))
- }
- columns, ok := res[0].([]string)
- if !ok {
- return 0, errors.Errorf("expected []string but got %t", res[0])
- }
- rowList, ok := res[2].([]any)
- if !ok {
- return 0, errors.Errorf("expected []any but got %t", res[2])
- }
- if len(rowList) < 1 {
- return 0, errors.Errorf("not found any data")
- }
-
- rowsIndex, err := getColumnIndex(columns, "rows")
- if err != nil {
- return 0, errors.Errorf("failed to find rows column")
- }
-
- for _, rowAny := range rowList {
- row, ok := rowAny.([]any)
- if !ok {
- return 0, errors.Errorf("expected []any but got %t", row)
- }
- switch col := row[rowsIndex].(type) {
- case int:
- return int64(col), nil
- case int32:
- return int64(col), nil
- case int64:
- return col, nil
- case string:
- v, err := strconv.ParseInt(col, 10, 64)
- if err != nil {
- return 0, errors.Errorf("expected int or int64 but got string(%s)", col)
- }
- return v, nil
- default:
- continue
- }
- }
-
- return 0, nil
-}
diff --git a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go
index fa88b062aa..e90388d107 100644
--- a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go
+++ b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go
@@ -3,7 +3,6 @@ package mysql
import (
"context"
"fmt"
- "strconv"
"strings"
"github.com/bytebase/omni/mysql/ast"
@@ -13,6 +12,7 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
+ mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"
mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
)
@@ -42,157 +42,68 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx adv
maxRow := int(numberPayload.Number)
driver := checkCtx.Driver
+ if driver == nil {
+ return nil, nil
+ }
title := checkCtx.Rule.Type.String()
var advice []*storepb.Advice
- explainCount := 0
-
- if driver != nil {
- for _, stmt := range checkCtx.ParsedStatements {
- if stmt.AST == nil {
- continue
- }
- node, ok := mysqlparser.GetOmniNode(stmt.AST)
- if !ok {
- continue
- }
-
- var limit *ast.Limit
- switch stmt := node.(type) {
- case *ast.UpdateStmt:
- limit = stmt.Limit
- case *ast.DeleteStmt:
- limit = stmt.Limit
- default:
- continue
- }
-
- baseLine := stmt.BaseLine()
- text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"
- line := baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line)
+ var explains advisor.ExplainBudget
- explainCount++
- // Prefer the JSON query plan: the tabular EXPLAIN's first "rows" value may
- // be the scan estimate of a driving table unrelated to the DML target
- // (BYT-9858). Fall back to the tabular heuristic when the JSON plan yields
- // no flagged target node.
- var rowCount int64
- counted := false
- if res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN FORMAT=JSON %s", text)); err == nil {
- rowCount, counted = getRowsFromJSONPlan(res)
- }
- if !counted {
- res, err := advisor.Query(ctx, advisor.QueryContext{}, driver, storepb.Engine_MYSQL, fmt.Sprintf("EXPLAIN %s", text))
- if err != nil {
- advice = append(advice, &storepb.Advice{
- Status: level,
- Code: code.StatementAffectedRowExceedsLimit.Int32(),
- Title: title,
- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(line),
- })
- } else if rowCount, err = getRows(res); err != nil {
- advice = append(advice, &storepb.Advice{
- Status: level,
- Code: code.Internal.Int32(),
- Title: title,
- Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(line),
- })
- } else {
- counted = true
- }
- }
- if counted {
- if rowCount = capRowsByLimit(rowCount, limit); rowCount > int64(maxRow) {
- advice = append(advice, &storepb.Advice{
- Status: level,
- Code: code.StatementAffectedRowExceedsLimit.Int32(),
- Title: title,
- Content: fmt.Sprintf("\"%s\" affected %d rows (estimated). The count exceeds %d.", text, rowCount, maxRow),
- StartPosition: common.ConvertANTLRLineToPosition(line),
- })
- }
- }
-
- if explainCount >= common.MaximumLintExplainSize {
- break
- }
+ for _, stmt := range checkCtx.ParsedStatements {
+ if stmt.AST == nil {
+ continue
}
- }
-
- return advice, nil
-}
-
-// getRowsFromJSONPlan extracts the affected-row estimate from an
-// `EXPLAIN FORMAT=JSON` query result, whose single column holds the JSON plan.
-func getRowsFromJSONPlan(res []any) (int64, bool) {
- if len(res) != 3 {
- return 0, false
- }
- rowList, ok := res[2].([]any)
- if !ok {
- return 0, false
- }
- var plan strings.Builder
- for _, rowAny := range rowList {
- row, ok := rowAny.([]any)
+ node, ok := mysqlparser.GetOmniNode(stmt.AST)
if !ok {
- return 0, false
+ continue
}
- for _, col := range row {
- if s, ok := col.(string); ok {
- plan.WriteString(s)
- }
+ switch node.(type) {
+ case *ast.UpdateStmt, *ast.DeleteStmt:
+ default:
+ continue
}
- }
- return mysqlparser.GetEstimatedAffectedRowsFromExplainJSON(plan.String())
-}
-func getRows(res []any) (int64, error) {
- // the res struct is []any{columnName, columnTable, rowDataList}
- if len(res) != 3 {
- return 0, errors.Errorf("expected 3 but got %d", len(res))
- }
- columns, ok := res[0].([]string)
- if !ok {
- return 0, errors.Errorf("expected []string but got %t", res[0])
- }
- rowList, ok := res[2].([]any)
- if !ok {
- return 0, errors.Errorf("expected []any but got %t", res[2])
- }
- if len(rowList) < 1 {
- return 0, errors.Errorf("not found any data")
- }
+ baseLine := stmt.BaseLine()
+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";") + ";"
+ position := common.ConvertANTLRLineToPosition(baseLine + int(mysqlparser.ByteOffsetToRunePosition(stmt.Text, contentStartIndex(stmt.Text)).Line))
- rowsIndex, err := getColumnIndex(columns, "rows")
- if err != nil {
- return 0, errors.Errorf("failed to find rows column")
- }
-
- for _, rowAny := range rowList {
- row, ok := rowAny.([]any)
- if !ok {
- return 0, errors.Errorf("expected []any but got %t", row)
+ if !explains.Spend(position) {
+ continue
}
- switch col := row[rowsIndex].(type) {
- case int:
- return int64(col), nil
- case int32:
- return int64(col), nil
- case int64:
- return col, nil
- case string:
- v, err := strconv.ParseInt(col, 10, 64)
- if err != nil {
- return 0, errors.Errorf("expected int or int64 but got string(%s)", col)
- }
- return v, nil
- default:
+ query := mysqlparser.AffectedRowsQuery(node, stmt.Text)
+ plan, err := mysqldriver.ExplainJSON(ctx, driver, query)
+ if err != nil {
+ advice = append(advice, &storepb.Advice{
+ Status: level,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("\"%s\" dry runs failed: %s", text, err.Error()),
+ StartPosition: position,
+ })
continue
}
+ rowCount, err := mysqldriver.EstimateAffectedRows(ctx, driver, node, query, plan)
+ if err != nil {
+ advice = append(advice, &storepb.Advice{
+ Status: level,
+ Code: code.Internal.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),
+ StartPosition: position,
+ })
+ continue
+ }
+ if rowCount > int64(maxRow) {
+ advice = append(advice, &storepb.Advice{
+ Status: level,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("\"%s\" affected %d rows (estimated). The count exceeds %d.", text, rowCount, maxRow),
+ StartPosition: position,
+ })
+ }
}
- return 0, nil
+ return explains.AppendSkippedAdvice(advice, title, code.StatementAffectedRowExceedsLimit), nil
}
diff --git a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go
index eae8b96c4f..ba3d970d82 100644
--- a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go
+++ b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit.go
@@ -61,25 +61,18 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contex
continue
}
checker.checkInsert(stmt.Text, stmt.BaseLine(), insert)
- if checker.explainCount >= common.MaximumLintExplainSize {
- break
- }
}
- return checker.generateAdvice()
+ return checker.explains.AppendSkippedAdvice(checker.adviceList, checker.title, code.InsertTooManyRows), nil
}
type insertRowLimitChecker struct {
- adviceList []*storepb.Advice
- level storepb.Advice_Status
- title string
- maxRow int
- driver *sql.DB
- ctx context.Context
- explainCount int
-}
-
-func (checker *insertRowLimitChecker) generateAdvice() ([]*storepb.Advice, error) {
- return checker.adviceList, nil
+ adviceList []*storepb.Advice
+ level storepb.Advice_Status
+ title string
+ maxRow int
+ driver *sql.DB
+ ctx context.Context
+ explains advisor.ExplainBudget
}
func (checker *insertRowLimitChecker) checkInsert(text string, baseLine int, stmt *ast.InsertStmt) {
@@ -94,8 +87,10 @@ func (checker *insertRowLimitChecker) handleInsertQueryExpression(text string, l
if checker.driver == nil {
return
}
+ if !checker.explains.Spend(common.ConvertANTLRLineToPosition(line)) {
+ return
+ }
- checker.explainCount++
res, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_OCEANBASE, fmt.Sprintf("EXPLAIN format=json %s", text))
if err != nil {
checker.adviceList = append(checker.adviceList, &storepb.Advice{
diff --git a/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.go b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.go
new file mode 100644
index 0000000000..bc7ce0b78d
--- /dev/null
+++ b/backend/plugin/advisor/oceanbase/advisor_insert_row_limit_test.go
@@ -0,0 +1,32 @@
+package oceanbase
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+func TestInsertRowLimitAdvisorKeepsCountingValuesBeyondExplainLimit(t *testing.T) {
+ const insertPlan = `{"ID":0,"OPERATOR":"DISTRIBUTED INSERT","NAME":"","EST.ROWS":100,"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t","EST.ROWS":100}}`
+ statement := strings.Repeat("INSERT INTO t2 SELECT * FROM t;\n", common.MaximumLintExplainSize+1) +
+ "INSERT INTO t2 (id) VALUES (1), (2), (3), (4), (5), (6);"
+
+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, insertPlan, statement)
+ require.Len(t, adviceList, common.MaximumLintExplainSize+2)
+ for _, advice := range adviceList[:common.MaximumLintExplainSize] {
+ require.Contains(t, advice.Content, "inserts 100 rows. The count exceeds 5.")
+ }
+ require.Contains(t, adviceList[common.MaximumLintExplainSize].Content, "inserts 6 rows. The count exceeds 5.")
+ warning := adviceList[common.MaximumLintExplainSize+1]
+ require.Equal(t, storepb.Advice_WARNING, warning.Status)
+ require.Equal(t, code.InsertTooManyRows.Int32(), warning.Code)
+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT.String(), warning.Title)
+ require.Equal(t, "Only the first 10 statements were estimated; 1 more were not checked against the row limit.", warning.Content)
+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())
+ require.Equal(t, common.MaximumLintExplainSize, testOceanBaseExplainCount)
+}
diff --git a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go
index ab15b10e74..40c836ecf5 100644
--- a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go
+++ b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit.go
@@ -62,27 +62,26 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx adv
checker.handleStmt(stmt.Text, omniLine(stmt.BaseLine(), stmt.Text, n.Loc))
default:
}
- if checker.explainCount >= common.MaximumLintExplainSize {
- break
- }
}
}
- return checker.adviceList, nil
+ return checker.explains.AppendSkippedAdvice(checker.adviceList, checker.title, code.StatementAffectedRowExceedsLimit), nil
}
type statementAffectedRowLimitChecker struct {
- adviceList []*storepb.Advice
- level storepb.Advice_Status
- title string
- maxRow int
- driver *sql.DB
- ctx context.Context
- explainCount int
+ adviceList []*storepb.Advice
+ level storepb.Advice_Status
+ title string
+ maxRow int
+ driver *sql.DB
+ ctx context.Context
+ explains advisor.ExplainBudget
}
func (checker *statementAffectedRowLimitChecker) handleStmt(text string, lineNumber int) {
- checker.explainCount++
+ if !checker.explains.Spend(common.ConvertANTLRLineToPosition(lineNumber)) {
+ return
+ }
res, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_OCEANBASE, fmt.Sprintf("EXPLAIN format=json %s", text))
if err != nil {
checker.adviceList = append(checker.adviceList, &storepb.Advice{
diff --git a/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.go b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.go
new file mode 100644
index 0000000000..032b7788cc
--- /dev/null
+++ b/backend/plugin/advisor/oceanbase/advisor_statement_affected_row_limit_test.go
@@ -0,0 +1,43 @@
+package oceanbase
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+func TestStatementAffectedRowLimitAdvisor(t *testing.T) {
+ const updatePlan = `{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":"","EST.ROWS":1000,"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t","EST.ROWS":1000}}`
+
+ t.Run("estimate over the limit", func(t *testing.T) {
+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, updatePlan, "UPDATE t SET c = 1;")
+ require.Len(t, adviceList, 1)
+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), adviceList[0].Code)
+ require.Equal(t, `"UPDATE t SET c = 1;" affected 1000 rows (estimated). The count exceeds 5.`, adviceList[0].Content)
+ })
+
+ t.Run("missing estimate", func(t *testing.T) {
+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, `{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":""}`, "UPDATE t SET c = 1;")
+ require.Len(t, adviceList, 1)
+ require.Equal(t, code.Internal.Int32(), adviceList[0].Code)
+ require.Contains(t, adviceList[0].Content, `operator "DISTRIBUTED UPDATE" has no EST.ROWS`)
+ })
+
+ t.Run("statements beyond the EXPLAIN limit", func(t *testing.T) {
+ statement := strings.Repeat("UPDATE t SET c = 1;\n", common.MaximumLintExplainSize+2)
+ adviceList := checkTestRowLimitRule(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, updatePlan, statement)
+ require.Len(t, adviceList, common.MaximumLintExplainSize+1)
+ warning := adviceList[common.MaximumLintExplainSize]
+ require.Equal(t, storepb.Advice_WARNING, warning.Status)
+ require.Equal(t, code.StatementAffectedRowExceedsLimit.Int32(), warning.Code)
+ require.Equal(t, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT.String(), warning.Title)
+ require.Equal(t, "Only the first 10 statements were estimated; 2 more were not checked against the row limit.", warning.Content)
+ require.Equal(t, int32(common.MaximumLintExplainSize+1), warning.StartPosition.GetLine())
+ require.Equal(t, common.MaximumLintExplainSize, testOceanBaseExplainCount)
+ })
+}
diff --git a/backend/plugin/advisor/oceanbase/common.go b/backend/plugin/advisor/oceanbase/common.go
index 858ab49da5..360018e48a 100644
--- a/backend/plugin/advisor/oceanbase/common.go
+++ b/backend/plugin/advisor/oceanbase/common.go
@@ -1,36 +1,26 @@
package oceanbase
import (
- "encoding/json"
+ "strings"
"github.com/pkg/errors"
-)
-// ExplainJSON represents the JSON structure returned by EXPLAIN format=json
-// We only care about the EST.ROWS field for row count estimation
-type ExplainJSON struct {
- EstRows int64 `json:"EST.ROWS"`
-}
+ mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
+)
-// getEstimatedRowsFromJSON extracts the estimated row count from OceanBase EXPLAIN format=json result
+// getEstimatedRowsFromJSON extracts the estimated row count from an OceanBase
+// `EXPLAIN FORMAT=JSON` result shaped as []any{columnNames, columnTypes, rows}.
func getEstimatedRowsFromJSON(res []any) (int64, error) {
- // For EXPLAIN format=json, OceanBase returns JSON data
- // The res struct is []any{columnName, columnTable, rowDataList}
if len(res) < 3 {
return 0, errors.Errorf("expected at least 3 elements but got %d", len(res))
}
-
rowList, ok := res[2].([]any)
if !ok {
return 0, errors.Errorf("expected []any for row data but got %T", res[2])
}
- if len(rowList) == 0 {
- return 0, errors.Errorf("no data returned from EXPLAIN")
- }
- // OceanBase might return JSON data split across multiple elements
- // We need to concatenate them to form a valid JSON string
- var jsonStr string
+ // OceanBase splits the JSON plan across rows.
+ var plan strings.Builder
for _, rowAny := range rowList {
row, ok := rowAny.([]any)
if !ok {
@@ -38,20 +28,9 @@ func getEstimatedRowsFromJSON(res []any) (int64, error) {
}
for _, cellAny := range row {
if cell, ok := cellAny.(string); ok {
- jsonStr += cell
+ plan.WriteString(cell)
}
}
}
-
- if jsonStr == "" {
- return 0, errors.Errorf("no JSON data found in EXPLAIN result")
- }
-
- var explainData ExplainJSON
- if err := json.Unmarshal([]byte(jsonStr), &explainData); err != nil {
- return 0, errors.Errorf("failed to parse JSON: %v", err)
- }
-
- // Return the estimated rows from the JSON response
- return explainData.EstRows, nil
+ return mysqlparser.EstimateAffectedRowsFromOceanBaseExplainJSON(plan.String())
}
diff --git a/backend/plugin/advisor/oceanbase/common_test.go b/backend/plugin/advisor/oceanbase/common_test.go
new file mode 100644
index 0000000000..73d1ddb6fe
--- /dev/null
+++ b/backend/plugin/advisor/oceanbase/common_test.go
@@ -0,0 +1,133 @@
+package oceanbase
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/component/sheet"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor"
+)
+
+var (
+ // testOceanBaseExplainPlan is the plan the fake OceanBase server returns for every EXPLAIN.
+ testOceanBaseExplainPlan string
+ // testOceanBaseExplainCount counts the EXPLAIN statements the fake server receives.
+ testOceanBaseExplainCount int
+)
+
+func init() {
+ sql.Register("test_oceanbase_advisor_explain", testOceanBaseExplainDriver{})
+}
+
+type testOceanBaseExplainDriver struct{}
+
+func (testOceanBaseExplainDriver) Open(string) (driver.Conn, error) {
+ return testOceanBaseExplainConn{}, nil
+}
+
+type testOceanBaseExplainConn struct{}
+
+func (testOceanBaseExplainConn) Prepare(string) (driver.Stmt, error) {
+ return nil, driver.ErrSkip
+}
+
+func (testOceanBaseExplainConn) Close() error {
+ return nil
+}
+
+func (testOceanBaseExplainConn) Begin() (driver.Tx, error) {
+ return testOceanBaseExplainTx{}, nil
+}
+
+func (testOceanBaseExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+ if strings.HasPrefix(query, "EXPLAIN ") {
+ testOceanBaseExplainCount++
+ }
+ return &testOceanBaseExplainRows{rows: [][]driver.Value{{testOceanBaseExplainPlan}}}, nil
+}
+
+type testOceanBaseExplainTx struct{}
+
+func (testOceanBaseExplainTx) Commit() error {
+ return nil
+}
+
+func (testOceanBaseExplainTx) Rollback() error {
+ return nil
+}
+
+type testOceanBaseExplainRows struct {
+ rows [][]driver.Value
+ idx int
+}
+
+func (*testOceanBaseExplainRows) Columns() []string {
+ return []string{"Query Plan"}
+}
+
+func (*testOceanBaseExplainRows) Close() error {
+ return nil
+}
+
+func (r *testOceanBaseExplainRows) Next(dest []driver.Value) error {
+ if r.idx >= len(r.rows) {
+ return io.EOF
+ }
+ copy(dest, r.rows[r.idx])
+ r.idx++
+ return nil
+}
+
+// checkTestRowLimitRule runs one OceanBase row limit rule with a limit of 5 against the fake
+// server, which answers every EXPLAIN with plan.
+func checkTestRowLimitRule(t *testing.T, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, plan, statement string) []*storepb.Advice {
+ t.Helper()
+ testOceanBaseExplainPlan = plan
+ testOceanBaseExplainCount = 0
+ db, err := sql.Open("test_oceanbase_advisor_explain", "")
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = db.Close() })
+
+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{
+ {
+ Type: ruleType,
+ Level: level,
+ Engine: storepb.Engine_OCEANBASE,
+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: 5}},
+ },
+ }, advisor.Context{
+ DBType: storepb.Engine_OCEANBASE,
+ Driver: db,
+ NoAppendBuiltin: true,
+ })
+ require.NoError(t, err)
+ return adviceList
+}
+
+func TestGetEstimatedRowsFromJSON(t *testing.T) {
+ // OceanBase splits the JSON plan across rows, and EST.ROWS may be fractional.
+ rows, err := getEstimatedRowsFromJSON([]any{
+ []string{"Query Plan"},
+ []string{"VARCHAR"},
+ []any{
+ []any{`{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE",`},
+ []any{`"NAME":"","EST.ROWS":41.6,"output":""}`},
+ },
+ })
+ require.NoError(t, err)
+ require.Equal(t, int64(42), rows)
+
+ _, err = getEstimatedRowsFromJSON([]any{
+ []string{"Query Plan"},
+ []string{"VARCHAR"},
+ []any{[]any{`{"ID":0,"OPERATOR":"DISTRIBUTED UPDATE","NAME":""}`}},
+ })
+ require.ErrorContains(t, err, `operator "DISTRIBUTED UPDATE" has no EST.ROWS`)
+}
diff --git a/backend/plugin/advisor/pg/advisor_insert_row_limit.go b/backend/plugin/advisor/pg/advisor_insert_row_limit.go
index fbc561527f..4bedb8fc9f 100644
--- a/backend/plugin/advisor/pg/advisor_insert_row_limit.go
+++ b/backend/plugin/advisor/pg/advisor_insert_row_limit.go
@@ -4,15 +4,16 @@ import (
"context"
"database/sql"
"fmt"
+ "strings"
"github.com/pkg/errors"
"github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
advisorcode "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
var (
@@ -53,18 +54,19 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Contex
TenantMode: checkCtx.TenantMode,
}
- return RunRules(checkCtx.ParsedStatements, []OmniRule{rule}), nil
+ adviceList := RunRules(checkCtx.ParsedStatements, []OmniRule{rule})
+ return rule.explains.AppendSkippedAdvice(adviceList, rule.Title, advisorcode.InsertTooManyRows), nil
}
type insertRowLimitRule struct {
OmniBaseRule
- maxRow int
- driver *sql.DB
- ctx context.Context
- explainCount int
- preExecutions []string
- TenantMode bool
+ maxRow int
+ driver *sql.DB
+ ctx context.Context
+ explains advisor.ExplainBudget
+ settings sessionSettings
+ TenantMode bool
}
func (*insertRowLimitRule) Name() string {
@@ -72,21 +74,19 @@ func (*insertRowLimitRule) Name() string {
}
func (r *insertRowLimitRule) OnStatement(node ast.Node) {
+ r.settings.add(node, r.TrimmedStmtText())
+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)
switch n := node.(type) {
- case *ast.VariableSetStmt:
- if omniIsRoleOrSearchPathSet(n) {
- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())
- }
case *ast.InsertStmt:
- r.checkInsert(n)
+ r.checkInsert(n, text)
default:
}
}
-func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {
+func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt, text string) {
code := advisorcode.Ok
rows := int64(0)
- statementText := r.TrimmedStmtText()
+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")
// Count VALUES rows if this is INSERT ... VALUES.
if sel, ok := ins.SelectStmt.(*ast.SelectStmt); ok && sel.ValuesLists != nil {
@@ -97,15 +97,14 @@ func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {
}
} else if ins.SelectStmt != nil && r.driver != nil {
// For INSERT ... SELECT, use EXPLAIN.
- if r.explainCount >= common.MaximumLintExplainSize {
+ if !r.explains.Spend(&storepb.Position{Line: r.ContentStartLine() + int32(r.BaseLine)}) {
return
}
- r.explainCount++
res, err := advisor.Query(r.ctx, advisor.QueryContext{
TenantMode: r.TenantMode,
- PreExecutions: r.preExecutions,
- }, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))
+ PreExecutions: r.settings.statements(),
+ }, r.driver, storepb.Engine_POSTGRES, getExplainSQL(statementText))
if err != nil {
r.AddAdvice(&storepb.Advice{
@@ -121,7 +120,7 @@ func (r *insertRowLimitRule) checkInsert(ins *ast.InsertStmt) {
return
}
- rowCount, err := getAffectedRows(res)
+ rowCount, err := getInsertedRows(res)
if err != nil {
r.AddAdvice(&storepb.Advice{
Status: r.Level,
diff --git a/backend/plugin/advisor/pg/advisor_insert_row_limit_test.go b/backend/plugin/advisor/pg/advisor_insert_row_limit_test.go
new file mode 100644
index 0000000000..e43a04a595
--- /dev/null
+++ b/backend/plugin/advisor/pg/advisor_insert_row_limit_test.go
@@ -0,0 +1,126 @@
+package pg
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+func TestInsertRowLimit(t *testing.T) {
+ const title = "STATEMENT_INSERT_ROW_LIMIT"
+ for _, tc := range []struct {
+ name string
+ statement string
+ plans map[string]string
+ wantExplained []string
+ want []*storepb.Advice
+ }{
+ {
+ name: "insert select on conflict",
+ statement: "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING;",
+ plans: map[string]string{
+ "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING": modifyTablePlan(1000),
+ },
+ wantExplained: []string{"INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.InsertTooManyRows.Int32(),
+ Title: title,
+ Content: `The statement "INSERT INTO big_pk SELECT * FROM big WHERE s_id <= 10 ON CONFLICT DO NOTHING" inserts 1000 rows. The count exceeds 100.`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ // The CTE's 5000 deleted rows are not inserted rows.
+ name: "insert with a data-modifying CTE counts only the inserted rows",
+ statement: "WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1;",
+ plans: map[string]string{
+ "WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1": `[{"Plan": {"Node Type": "ModifyTable", "Operation": "Insert", "Plan Rows": 0, "Plans": [
+ {"Node Type": "ModifyTable", "Operation": "Delete", "Parent Relationship": "InitPlan", "Plan Rows": 5000, "Plans": [
+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}
+ ]},
+ {"Node Type": "Limit", "Parent Relationship": "Outer", "Plan Rows": 1}
+ ]}}]`,
+ },
+ wantExplained: []string{"WITH d AS (DELETE FROM big RETURNING id) INSERT INTO archive SELECT id FROM d LIMIT 1"},
+ },
+ {
+ name: "insert select within the limit",
+ statement: "INSERT INTO t2 SELECT * FROM t LIMIT 30;",
+ plans: map[string]string{"INSERT INTO t2 SELECT * FROM t LIMIT 30": modifyTablePlan(30)},
+ wantExplained: []string{"INSERT INTO t2 SELECT * FROM t LIMIT 30"},
+ },
+ {
+ name: "dry run failure",
+ statement: "INSERT INTO missing SELECT * FROM t;",
+ wantExplained: []string{"INSERT INTO missing SELECT * FROM t"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.InsertTooManyRows.Int32(),
+ Title: title,
+ Content: `"INSERT INTO missing SELECT * FROM t" dry runs failed: relation in "INSERT INTO missing SELECT * FROM t" does not exist`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "plan without a ModifyTable node",
+ statement: "INSERT INTO t2 SELECT * FROM t;",
+ plans: map[string]string{"INSERT INTO t2 SELECT * FROM t": "Insert on t2 (cost=0.00..17.00 rows=0 width=0)"},
+ wantExplained: []string{"INSERT INTO t2 SELECT * FROM t"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.Internal.Int32(),
+ Title: title,
+ Content: `failed to get row count for "INSERT INTO t2 SELECT * FROM t": failed to parse the EXPLAIN (FORMAT JSON) output: invalid character 'I' looking for beginning of value`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "insert values is counted without explain",
+ statement: "INSERT INTO t2 VALUES (1), (2), (3) ON CONFLICT DO NOTHING;",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ db, fake := newExplainFakeDB(t, tc.plans)
+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_WARNING, 100, tc.statement)
+ require.Equal(t, tc.wantExplained, fake.explainedStatements())
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestInsertRowLimitReportsStatementsBeyondExplainLimit(t *testing.T) {
+ plans := map[string]string{}
+ var statements []string
+ for i := range 11 {
+ statement := fmt.Sprintf("INSERT INTO t2 SELECT * FROM t WHERE id = %d", i)
+ plans[statement] = modifyTablePlan(1)
+ statements = append(statements, statement+";")
+ }
+ statements = append(statements, "INSERT INTO t2 VALUES (1), (2), (3);", "INSERT INTO t2 SELECT * FROM t;")
+ db, fake := newExplainFakeDB(t, plans)
+
+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT, storepb.SQLReviewRule_ERROR, 2, strings.Join(statements, "\n"))
+ require.Len(t, fake.explainedStatements(), 10)
+ require.Equal(t, []*storepb.Advice{
+ {
+ Status: storepb.Advice_ERROR,
+ Code: code.InsertTooManyRows.Int32(),
+ Title: "STATEMENT_INSERT_ROW_LIMIT",
+ Content: `The statement "INSERT INTO t2 VALUES (1), (2), (3)" inserts 3 rows. The count exceeds 2.`,
+ StartPosition: &storepb.Position{Line: 12},
+ },
+ {
+ Status: storepb.Advice_WARNING,
+ Code: code.InsertTooManyRows.Int32(),
+ Title: "STATEMENT_INSERT_ROW_LIMIT",
+ Content: "Only the first 10 statements were estimated; 2 more were not checked against the row limit.",
+ StartPosition: &storepb.Position{Line: 11},
+ },
+ }, got)
+}
diff --git a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go
index da898f917a..66fae4d0d5 100644
--- a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go
+++ b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit.go
@@ -4,15 +4,16 @@ import (
"context"
"database/sql"
"fmt"
+ "strings"
"github.com/pkg/errors"
"github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
var (
@@ -23,11 +24,11 @@ func init() {
advisor.Register(storepb.Engine_POSTGRES, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, &StatementAffectedRowLimitAdvisor{})
}
-// StatementAffectedRowLimitAdvisor is the advisor checking for UPDATE/DELETE affected row limit.
+// StatementAffectedRowLimitAdvisor is the advisor checking for UPDATE/DELETE/MERGE affected row limit.
type StatementAffectedRowLimitAdvisor struct {
}
-// Check checks for UPDATE/DELETE affected row limit.
+// Check checks for UPDATE/DELETE/MERGE affected row limit, including data-modifying CTEs.
func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {
level, err := advisor.NewStatusBySQLReviewRuleLevel(checkCtx.Rule.Level)
if err != nil {
@@ -54,17 +55,18 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx adv
tenantMode: checkCtx.TenantMode,
}
- return RunRules(checkCtx.ParsedStatements, []OmniRule{rule}), nil
+ adviceList := RunRules(checkCtx.ParsedStatements, []OmniRule{rule})
+ return rule.explains.AppendSkippedAdvice(adviceList, rule.Title, code.StatementAffectedRowExceedsLimit), nil
}
type statementAffectedRowLimitRule struct {
OmniBaseRule
- maxRow int
- driver *sql.DB
- ctx context.Context
- explainCount int
- preExecutions []string
- tenantMode bool
+ maxRow int
+ driver *sql.DB
+ ctx context.Context
+ explains advisor.ExplainBudget
+ settings sessionSettings
+ tenantMode bool
}
func (*statementAffectedRowLimitRule) Name() string {
@@ -72,35 +74,43 @@ func (*statementAffectedRowLimitRule) Name() string {
}
func (r *statementAffectedRowLimitRule) OnStatement(node ast.Node) {
+ r.settings.add(node, r.TrimmedStmtText())
+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)
switch n := node.(type) {
- case *ast.VariableSetStmt:
- if omniIsRoleOrSearchPathSet(n) {
- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())
+ case *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:
+ r.checkAffectedRows(text)
+ case *ast.SelectStmt:
+ if hasDataModifyingCTE(n.WithClause) {
+ r.checkAffectedRows(text)
+ }
+ case *ast.InsertStmt:
+ if hasDataModifyingCTE(n.WithClause) {
+ r.checkAffectedRows(text)
+ }
+ case *ast.CreateTableAsStmt:
+ if query, ok := n.Query.(*ast.SelectStmt); ok && hasDataModifyingCTE(query.WithClause) {
+ r.checkAffectedRows(text)
}
- case *ast.UpdateStmt, *ast.DeleteStmt:
- r.checkAffectedRows()
default:
}
}
-func (r *statementAffectedRowLimitRule) checkAffectedRows() {
- if r.explainCount >= common.MaximumLintExplainSize {
+func (r *statementAffectedRowLimitRule) checkAffectedRows(text string) {
+ if !r.explains.Spend(&storepb.Position{Line: r.ContentStartLine() + int32(r.BaseLine)}) {
return
}
- r.explainCount++
-
- statementText := r.TrimmedStmtText()
+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")
res, err := advisor.Query(r.ctx, advisor.QueryContext{
TenantMode: r.tenantMode,
- PreExecutions: r.preExecutions,
- }, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))
+ PreExecutions: r.settings.statements(),
+ }, r.driver, storepb.Engine_POSTGRES, getExplainSQL(statementText))
if err != nil {
r.AddAdvice(&storepb.Advice{
Status: r.Level,
- Code: code.InsertTooManyRows.Int32(),
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
Title: r.Title,
Content: fmt.Sprintf("\"%s\" dry runs failed: %s", statementText, err.Error()),
StartPosition: &storepb.Position{
diff --git a/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.go b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.go
new file mode 100644
index 0000000000..4bd4c3caf8
--- /dev/null
+++ b/backend/plugin/advisor/pg/advisor_statement_affected_row_limit_test.go
@@ -0,0 +1,174 @@
+package pg
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+func TestStatementAffectedRowLimit(t *testing.T) {
+ const title = "STATEMENT_AFFECTED_ROW_LIMIT"
+ for _, tc := range []struct {
+ name string
+ statement string
+ plans map[string]string
+ wantExplained []string
+ want []*storepb.Advice
+ }{
+ {
+ name: "estimate from the subplan feeding the update, not its init plan",
+ statement: "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag);",
+ plans: map[string]string{
+ "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [
+ {"Node Type": "Aggregate", "Parent Relationship": "InitPlan", "Subplan Name": "InitPlan 1", "Plan Rows": 1},
+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 3333}
+ ]}}]`,
+ },
+ wantExplained: []string{"UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `The statement "UPDATE big SET v = 1 WHERE s_id <= (SELECT count(*) FROM s WHERE flag)" affected 3333 rows (estimated). The count exceeds 1000.`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "merge",
+ statement: "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1;",
+ plans: map[string]string{
+ "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1": modifyTablePlan(10000),
+ },
+ wantExplained: []string{"MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `The statement "MERGE INTO big USING s ON big.s_id = s.id WHEN MATCHED THEN UPDATE SET v = 1" affected 10000 rows (estimated). The count exceeds 1000.`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "data-modifying CTEs",
+ statement: "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d;\nWITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d;\nCREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d;",
+ plans: map[string]string{
+ "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d": `[{"Plan": {"Node Type": "Aggregate", "Plan Rows": 1, "Plans": [
+ {"Node Type": "ModifyTable", "Parent Relationship": "InitPlan", "Subplan Name": "CTE d", "Plan Rows": 5000, "Plans": [
+ {"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}
+ ]},
+ {"Node Type": "CTE Scan", "Parent Relationship": "Outer", "Plan Rows": 5000}
+ ]}}]`,
+ "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d": modifyTablePlan(20000),
+ "CREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d": modifyTablePlan(100),
+ },
+ wantExplained: []string{
+ "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d",
+ "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d",
+ "CREATE TABLE big_moved AS WITH d AS (DELETE FROM big WHERE s_id = 1 RETURNING *) SELECT * FROM d",
+ },
+ want: []*storepb.Advice{
+ {
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `The statement "WITH d AS (DELETE FROM big WHERE s_id <= 50 RETURNING 1) SELECT count(*) FROM d" affected 5000 rows (estimated). The count exceeds 1000.`,
+ StartPosition: &storepb.Position{Line: 1},
+ },
+ {
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `The statement "WITH d AS (DELETE FROM big RETURNING *) INSERT INTO big_archive SELECT * FROM d" affected 20000 rows (estimated). The count exceeds 1000.`,
+ StartPosition: &storepb.Position{Line: 2},
+ },
+ },
+ },
+ {
+ name: "select, read-only CTE, and insert are not explained",
+ statement: "SELECT * FROM big;\nWITH x AS (SELECT id FROM s) SELECT * FROM x;\nINSERT INTO big_archive SELECT * FROM big;",
+ },
+ {
+ name: "explain analyze runs the statement it explains",
+ statement: "EXPLAIN (ANALYZE, BUFFERS) DELETE FROM big WHERE s_id <= 50;",
+ plans: map[string]string{
+ "DELETE FROM big WHERE s_id <= 50": modifyTablePlan(5000),
+ },
+ wantExplained: []string{"DELETE FROM big WHERE s_id <= 50"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `The statement "DELETE FROM big WHERE s_id <= 50" affected 5000 rows (estimated). The count exceeds 1000.`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "explain without analyze is not explained",
+ statement: "EXPLAIN (ANALYZE off) DELETE FROM big;",
+ },
+ {
+ name: "estimate within the limit",
+ statement: "DELETE FROM big WHERE id = 1;",
+ plans: map[string]string{"DELETE FROM big WHERE id = 1": modifyTablePlan(1)},
+ wantExplained: []string{"DELETE FROM big WHERE id = 1"},
+ },
+ {
+ name: "dry run failure",
+ statement: "DELETE FROM missing;",
+ wantExplained: []string{"DELETE FROM missing"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: title,
+ Content: `"DELETE FROM missing" dry runs failed: relation in "DELETE FROM missing" does not exist`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ {
+ name: "plan without a ModifyTable node",
+ statement: "DELETE FROM big;",
+ plans: map[string]string{"DELETE FROM big": `[{"Plan": {"Node Type": "Seq Scan", "Plan Rows": 10000}}]`},
+ wantExplained: []string{"DELETE FROM big"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.Internal.Int32(),
+ Title: title,
+ Content: `failed to get row count for "DELETE FROM big": the plan has no ModifyTable node`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ db, fake := newExplainFakeDB(t, tc.plans)
+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_WARNING, 1000, tc.statement)
+ require.Equal(t, tc.wantExplained, fake.explainedStatements())
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestStatementAffectedRowLimitReportsStatementsBeyondExplainLimit(t *testing.T) {
+ plans := map[string]string{}
+ var statements []string
+ for i := range 12 {
+ statement := fmt.Sprintf("UPDATE t SET v = 'y' WHERE id = %d", i)
+ plans[statement] = modifyTablePlan(1)
+ statements = append(statements, statement+";")
+ }
+ db, fake := newExplainFakeDB(t, plans)
+
+ got := runRowLimitRule(t, db, storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT, storepb.SQLReviewRule_ERROR, 1000, strings.Join(statements, "\n"))
+ require.Len(t, fake.explainedStatements(), 10)
+ require.Equal(t, []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementAffectedRowExceedsLimit.Int32(),
+ Title: "STATEMENT_AFFECTED_ROW_LIMIT",
+ Content: "Only the first 10 statements were estimated; 2 more were not checked against the row limit.",
+ StartPosition: &storepb.Position{Line: 11},
+ }}, got)
+}
diff --git a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go
index 3844ee9347..5629aa3402 100644
--- a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go
+++ b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
+ "strings"
"github.com/bytebase/omni/pg/ast"
@@ -11,6 +12,7 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/advisor"
"github.com/bytebase/bytebase/backend/plugin/advisor/code"
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
var (
@@ -58,11 +60,11 @@ func (*StatementDMLDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Co
type statementDMLDryRunRule struct {
OmniBaseRule
- driver *sql.DB
- ctx context.Context
- explainCount int
- preExecutions []string
- tenantMode bool
+ driver *sql.DB
+ ctx context.Context
+ explainCount int
+ settings sessionSettings
+ tenantMode bool
}
// Name returns the rule name.
@@ -71,18 +73,20 @@ func (*statementDMLDryRunRule) Name() string {
}
func (r *statementDMLDryRunRule) OnStatement(node ast.Node) {
+ r.settings.add(node, r.TrimmedStmtText())
+ node, text := pgparser.UnwrapExplainAnalyze(node, r.StmtText)
switch n := node.(type) {
- case *ast.VariableSetStmt:
- if omniIsRoleOrSearchPathSet(n) {
- r.preExecutions = append(r.preExecutions, r.TrimmedStmtText())
+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:
+ r.checkDMLDryRun(text)
+ case *ast.SelectStmt:
+ if hasDataModifyingCTE(n.WithClause) {
+ r.checkDMLDryRun(text)
}
- case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:
- r.checkDMLDryRun()
default:
}
}
-func (r *statementDMLDryRunRule) checkDMLDryRun() {
+func (r *statementDMLDryRunRule) checkDMLDryRun(text string) {
// Check if we've hit the maximum number of EXPLAIN queries
if r.explainCount >= common.MaximumLintExplainSize {
return
@@ -90,12 +94,12 @@ func (r *statementDMLDryRunRule) checkDMLDryRun() {
r.explainCount++
- statementText := r.TrimmedStmtText()
+ statementText := strings.TrimRight(strings.TrimSpace(text), ";")
// Run EXPLAIN to perform dry run
_, err := advisor.Query(r.ctx, advisor.QueryContext{
TenantMode: r.tenantMode,
- PreExecutions: r.preExecutions,
+ PreExecutions: r.settings.statements(),
}, r.driver, storepb.Engine_POSTGRES, fmt.Sprintf("EXPLAIN %s", statementText))
if err != nil {
diff --git a/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.go b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.go
new file mode 100644
index 0000000000..99b6d0216f
--- /dev/null
+++ b/backend/plugin/advisor/pg/advisor_statement_dml_dry_run_test.go
@@ -0,0 +1,84 @@
+package pg
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/component/sheet"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+func TestStatementDMLDryRun(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ plans map[string]string
+ wantExplained []string
+ want []*storepb.Advice
+ }{
+ {
+ name: "insert, update, delete, and merge",
+ statement: "INSERT INTO t VALUES (1);\nUPDATE t SET v = 1;\nDELETE FROM t;\nMERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE;",
+ plans: map[string]string{
+ "INSERT INTO t VALUES (1)": modifyTablePlan(1),
+ "UPDATE t SET v = 1": modifyTablePlan(1),
+ "DELETE FROM t": modifyTablePlan(1),
+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE": modifyTablePlan(1),
+ },
+ wantExplained: []string{
+ "INSERT INTO t VALUES (1)",
+ "UPDATE t SET v = 1",
+ "DELETE FROM t",
+ "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE",
+ },
+ },
+ {
+ name: "data-modifying CTE",
+ statement: "WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d;",
+ plans: map[string]string{"WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d": modifyTablePlan(1)},
+ wantExplained: []string{"WITH d AS (DELETE FROM t RETURNING *) SELECT count(*) FROM d"},
+ },
+ {
+ name: "explain analyze runs the statement it explains",
+ statement: "EXPLAIN (ANALYZE, BUFFERS) INSERT INTO t VALUES (1);",
+ plans: map[string]string{"INSERT INTO t VALUES (1)": modifyTablePlan(1)},
+ wantExplained: []string{"INSERT INTO t VALUES (1)"},
+ },
+ {
+ name: "select and explain without analyze are not dry run",
+ statement: "SELECT * FROM t;\nEXPLAIN DELETE FROM t;",
+ },
+ {
+ name: "dry run failure",
+ statement: "MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE;",
+ wantExplained: []string{"MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE"},
+ want: []*storepb.Advice{{
+ Status: storepb.Advice_WARNING,
+ Code: code.StatementDMLDryRunFailed.Int32(),
+ Title: "STATEMENT_DML_DRY_RUN",
+ Content: `"MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE" dry runs failed: relation in "MERGE INTO missing USING s ON missing.id = s.id WHEN MATCHED THEN DELETE" does not exist`,
+ StartPosition: &storepb.Position{Line: 1},
+ }},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ db, fake := newExplainFakeDB(t, tc.plans)
+ got, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), tc.statement, []*storepb.SQLReviewRule{{
+ Type: storepb.SQLReviewRule_STATEMENT_DML_DRY_RUN,
+ Level: storepb.SQLReviewRule_WARNING,
+ Engine: storepb.Engine_POSTGRES,
+ }}, advisor.Context{
+ DBType: storepb.Engine_POSTGRES,
+ Driver: db,
+ NoAppendBuiltin: true,
+ })
+ require.NoError(t, err)
+ require.Equal(t, tc.wantExplained, fake.explainedStatements())
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
diff --git a/backend/plugin/advisor/pg/testutil_test.go b/backend/plugin/advisor/pg/testutil_test.go
new file mode 100644
index 0000000000..b9d17299d8
--- /dev/null
+++ b/backend/plugin/advisor/pg/testutil_test.go
@@ -0,0 +1,132 @@
+package pg
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "fmt"
+ "io"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/component/sheet"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor"
+)
+
+// explainFake stands in for PostgreSQL behind advisor.Query. It answers `EXPLAIN (FORMAT JSON)` or
+// `EXPLAIN` with the plan registered for the statement and fails to explain any other statement.
+type explainFake struct {
+ plans map[string]string
+
+ mu sync.Mutex
+ explained []string
+}
+
+func newExplainFakeDB(t *testing.T, plans map[string]string) (*sql.DB, *explainFake) {
+ t.Helper()
+ fake := &explainFake{plans: plans}
+ db := sql.OpenDB(fake)
+ t.Cleanup(func() { require.NoError(t, db.Close()) })
+ return db, fake
+}
+
+func (f *explainFake) Connect(context.Context) (driver.Conn, error) {
+ return explainFakeConn{fake: f}, nil
+}
+
+func (f *explainFake) Driver() driver.Driver {
+ return explainFakeDriver{fake: f}
+}
+
+func (f *explainFake) explainedStatements() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return f.explained
+}
+
+type explainFakeDriver struct{ fake *explainFake }
+
+func (d explainFakeDriver) Open(string) (driver.Conn, error) {
+ return explainFakeConn(d), nil
+}
+
+type explainFakeConn struct{ fake *explainFake }
+
+func (explainFakeConn) Prepare(query string) (driver.Stmt, error) {
+ return nil, errors.Errorf("unexpected prepare of %q", query)
+}
+
+func (explainFakeConn) Close() error { return nil }
+
+func (explainFakeConn) Begin() (driver.Tx, error) { return explainFakeTx{}, nil }
+
+func (explainFakeConn) ExecContext(context.Context, string, []driver.NamedValue) (driver.Result, error) {
+ return driver.ResultNoRows, nil
+}
+
+func (c explainFakeConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+ statement, ok := strings.CutPrefix(query, "EXPLAIN (FORMAT JSON) ")
+ if !ok {
+ statement, ok = strings.CutPrefix(query, "EXPLAIN ")
+ }
+ if !ok {
+ return nil, errors.Errorf("unexpected query %q", query)
+ }
+ c.fake.mu.Lock()
+ c.fake.explained = append(c.fake.explained, statement)
+ c.fake.mu.Unlock()
+ plan, ok := c.fake.plans[statement]
+ if !ok {
+ return nil, errors.Errorf("relation in %q does not exist", statement)
+ }
+ return &explainFakeRows{plan: plan}, nil
+}
+
+type explainFakeTx struct{}
+
+func (explainFakeTx) Commit() error { return nil }
+func (explainFakeTx) Rollback() error { return nil }
+
+type explainFakeRows struct {
+ plan string
+ done bool
+}
+
+func (*explainFakeRows) Columns() []string { return []string{"QUERY PLAN"} }
+
+func (*explainFakeRows) Close() error { return nil }
+
+func (r *explainFakeRows) Next(dest []driver.Value) error {
+ if r.done {
+ return io.EOF
+ }
+ dest[0] = r.plan
+ r.done = true
+ return nil
+}
+
+// modifyTablePlan returns a minimal `EXPLAIN (FORMAT JSON)` plan whose ModifyTable node is fed rows.
+func modifyTablePlan(rows int) string {
+ return fmt.Sprintf(`[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [{"Node Type": "Seq Scan", "Parent Relationship": "Outer", "Plan Rows": %d}]}}]`, rows)
+}
+
+func runRowLimitRule(t *testing.T, db *sql.DB, ruleType storepb.SQLReviewRule_Type, level storepb.SQLReviewRule_Level, limit int32, statement string) []*storepb.Advice {
+ t.Helper()
+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sheet.NewManager(), statement, []*storepb.SQLReviewRule{{
+ Type: ruleType,
+ Level: level,
+ Engine: storepb.Engine_POSTGRES,
+ Payload: &storepb.SQLReviewRule_NumberPayload{NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{Number: limit}},
+ }}, advisor.Context{
+ DBType: storepb.Engine_POSTGRES,
+ Driver: db,
+ NoAppendBuiltin: true,
+ })
+ require.NoError(t, err)
+ return adviceList
+}
diff --git a/backend/plugin/advisor/pg/utils.go b/backend/plugin/advisor/pg/utils.go
index 76d0a0c1fd..b7b5decb36 100644
--- a/backend/plugin/advisor/pg/utils.go
+++ b/backend/plugin/advisor/pg/utils.go
@@ -3,11 +3,13 @@ package pg
import (
"fmt"
"regexp"
- "strconv"
+ "slices"
"strings"
"github.com/bytebase/omni/pg/ast"
"github.com/pkg/errors"
+
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
func getTemplateRegexp(template string, templateList []string, tokens map[string]string) (*regexp.Regexp, error) {
@@ -28,45 +30,73 @@ func normalizeSchemaName(schemaName string) string {
return schemaName
}
-// getAffectedRows extracts the estimated row count from a PostgreSQL EXPLAIN result.
-func getAffectedRows(res []any) (int64, error) {
+// getExplainSQL returns the query whose result getExplainPlan reads.
+func getExplainSQL(statement string) string {
+ return fmt.Sprintf("EXPLAIN (FORMAT JSON) %s", statement)
+}
+
+// getExplainPlan returns the JSON plan from the advisor.Query result of a getExplainSQL query.
+func getExplainPlan(res []any) (string, error) {
// the res struct is []any{columnName, columnTable, rowDataList}
if len(res) != 3 {
- return 0, errors.Errorf("expected 3 but got %d", len(res))
+ return "", errors.Errorf("expected 3 but got %d", len(res))
}
rowList, ok := res[2].([]any)
if !ok {
- return 0, errors.Errorf("expected []any but got %t", res[2])
- }
- // EXPLAIN output has at least 2 rows
- if len(rowList) < 2 {
- return 0, errors.Errorf("not found any data")
+ return "", errors.Errorf("expected []any but got %T", res[2])
}
- // We need row 2
- rowTwo, ok := rowList[1].([]any)
- if !ok {
- return 0, errors.Errorf("expected []any but got %t", rowList[0])
+ if len(rowList) != 1 {
+ return "", errors.Errorf("expected one plan row but got %d", len(rowList))
}
- // PostgreSQL EXPLAIN result has one column
- if len(rowTwo) != 1 {
- return 0, errors.Errorf("expected one but got %d", len(rowTwo))
+ row, ok := rowList[0].([]any)
+ if !ok || len(row) != 1 {
+ return "", errors.Errorf("expected one plan column but got %v", rowList[0])
}
- // Get the string value
- text, ok := rowTwo[0].(string)
+ plan, ok := row[0].(string)
if !ok {
- return 0, errors.Errorf("expected string but got %t", rowTwo[0])
+ return "", errors.Errorf("expected string but got %T", row[0])
}
+ return plan, nil
+}
- rowsRegexp := regexp.MustCompile("rows=([0-9]+)")
- matches := rowsRegexp.FindStringSubmatch(text)
- if len(matches) != 2 {
- return 0, errors.Errorf("failed to find rows in %q", text)
+// getAffectedRows returns the estimated rows a statement and its data-modifying CTEs modify, from
+// the advisor.Query result of its getExplainSQL query.
+func getAffectedRows(res []any) (int64, error) {
+ plan, err := getExplainPlan(res)
+ if err != nil {
+ return 0, err
}
- value, err := strconv.ParseInt(matches[1], 10, 64)
+ return pgparser.GetEstimatedAffectedRowsFromExplainJSON(plan)
+}
+
+// getInsertedRows returns the estimated rows an INSERT adds, without the rows its data-modifying
+// CTEs change, from the advisor.Query result of its getExplainSQL query.
+func getInsertedRows(res []any) (int64, error) {
+ plan, err := getExplainPlan(res)
if err != nil {
- return 0, errors.Errorf("failed to get integer from %q", matches[1])
+ return 0, err
+ }
+ return pgparser.GetEstimatedInsertedRowsFromExplainJSON(plan)
+}
+
+// hasDataModifyingCTE reports whether a WITH clause contains INSERT, UPDATE, DELETE, or MERGE.
+// PostgreSQL only allows data-modifying statements in a top-level WITH.
+func hasDataModifyingCTE(with *ast.WithClause) bool {
+ if with == nil || with.Ctes == nil {
+ return false
+ }
+ for _, item := range with.Ctes.Items {
+ cte, ok := item.(*ast.CommonTableExpr)
+ if !ok {
+ continue
+ }
+ switch cte.Ctequery.(type) {
+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:
+ return true
+ default:
+ }
}
- return value, nil
+ return false
}
// nolint:unused
@@ -153,14 +183,78 @@ func omniAlterTableCmds(alter *ast.AlterTableStmt) []*ast.AlterTableCmd {
return cmds
}
-// nolint:unused
-// omniIsRoleOrSearchPathSet checks if a VariableSetStmt is SET ROLE or SET search_path.
+// sessionSettings holds, in order, the statements that set the role or search path, which an EXPLAIN
+// replays. A SET LOCAL setting lasts until its transaction ends, ROLLBACK also drops the settings of
+// its transaction, and DISCARD ALL drops every setting.
+type sessionSettings struct {
+ settings []sessionSetting
+ // transactionStart is the number of settings when the current transaction began.
+ transactionStart int
+ inTransaction bool
+}
+
+type sessionSetting struct {
+ name string
+ text string
+ local bool
+}
+
+// add records node, whose text is text, when it changes the settings.
+func (s *sessionSettings) add(node ast.Node, text string) {
+ switch n := node.(type) {
+ case *ast.VariableSetStmt:
+ if omniIsRoleOrSearchPathSet(n) {
+ if n.Kind == ast.VAR_SET_CURRENT && !n.IsLocal {
+ // SET ... FROM CURRENT keeps the value a SET LOCAL gave the setting after the transaction ends.
+ for i := range s.settings {
+ if strings.EqualFold(s.settings[i].name, n.Name) {
+ s.settings[i].local = false
+ }
+ }
+ }
+ s.settings = append(s.settings, sessionSetting{name: n.Name, text: text, local: n.IsLocal})
+ }
+ case *ast.DiscardStmt:
+ if n.Target == ast.DISCARD_ALL {
+ s.settings, s.transactionStart = nil, 0
+ }
+ case *ast.TransactionStmt:
+ switch n.Kind {
+ case ast.TRANS_STMT_BEGIN, ast.TRANS_STMT_START:
+ // A BEGIN inside a transaction only warns.
+ if !s.inTransaction {
+ s.transactionStart, s.inTransaction = len(s.settings), true
+ }
+ case ast.TRANS_STMT_COMMIT, ast.TRANS_STMT_ROLLBACK:
+ // A ROLLBACK outside a transaction only warns.
+ if n.Kind == ast.TRANS_STMT_ROLLBACK && s.inTransaction {
+ s.settings = s.settings[:s.transactionStart]
+ }
+ s.settings = slices.DeleteFunc(s.settings, func(setting sessionSetting) bool { return setting.local })
+ // COMMIT AND CHAIN and ROLLBACK AND CHAIN start the next transaction from here.
+ s.transactionStart, s.inTransaction = len(s.settings), n.Chain
+ default:
+ }
+ default:
+ }
+}
+
+func (s *sessionSettings) statements() []string {
+ var statements []string
+ for _, setting := range s.settings {
+ statements = append(statements, setting.text)
+ }
+ return statements
+}
+
+// omniIsRoleOrSearchPathSet checks if a VariableSetStmt sets the role or search path, including a
+// RESET ALL.
func omniIsRoleOrSearchPathSet(stmt *ast.VariableSetStmt) bool {
if stmt == nil {
return false
}
name := strings.ToLower(stmt.Name)
- return name == "role" || name == "search_path"
+ return name == "role" || name == "search_path" || stmt.Kind == ast.VAR_RESET_ALL
}
// nolint:unused
diff --git a/backend/plugin/advisor/pg/utils_test.go b/backend/plugin/advisor/pg/utils_test.go
index 9cb7a54b02..651f91453f 100644
--- a/backend/plugin/advisor/pg/utils_test.go
+++ b/backend/plugin/advisor/pg/utils_test.go
@@ -33,6 +33,11 @@ func TestOmniIsRoleOrSearchPathSet(t *testing.T) {
sql: "SET search_path = myschema, public; UPDATE t SET a = 1;",
want: []string{"SET search_path = myschema, public"},
},
+ {
+ name: "reset all",
+ sql: "SET search_path = myschema; RESET ALL; UPDATE t SET a = 1;",
+ want: []string{"SET search_path = myschema", "RESET ALL"},
+ },
{
name: "ignore unrelated set variable",
sql: "SET statement_timeout = '1s'; DELETE FROM t;",
@@ -65,3 +70,59 @@ func TestOmniIsRoleOrSearchPathSet(t *testing.T) {
})
}
}
+
+func TestSessionSettings(t *testing.T) {
+ var settings sessionSettings
+ add := func(statement string) {
+ statements, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)
+ require.NoError(t, err)
+ for _, stmt := range statements {
+ node, ok := pgparser.GetOmniNode(stmt.AST)
+ require.True(t, ok)
+ settings.add(node, strings.TrimRight(strings.TrimSpace(stmt.Text), ";"))
+ }
+ }
+
+ add(`SET search_path = z;
+DISCARD ALL;
+SET search_path = a;
+BEGIN;
+SET LOCAL search_path = b;
+SET ROLE r;
+COMMIT;
+BEGIN;
+SET search_path = c;
+ROLLBACK;
+BEGIN;
+SET search_path = d;
+COMMIT AND CHAIN;
+ROLLBACK;
+SET LOCAL search_path = e;
+SET statement_timeout = '1s';
+RESET search_path;`)
+ // The settings replay in statement order, so RESET follows the SET LOCAL it overrides.
+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "SET LOCAL search_path = e", "RESET search_path"}, settings.statements())
+
+ // A nested BEGIN only warns, so the ROLLBACK returns to the settings before the first BEGIN, and
+ // the transaction's end also ends the SET LOCAL.
+ add(`BEGIN;
+SET search_path = f;
+BEGIN;
+SET search_path = g;
+ROLLBACK;`)
+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path"}, settings.statements())
+
+ // A ROLLBACK outside a transaction only warns, so it keeps the session settings.
+ add(`SET search_path = h;
+SET LOCAL search_path = i;
+ROLLBACK;`)
+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path", "SET search_path = h"}, settings.statements())
+
+ // SET ... FROM CURRENT keeps the SET LOCAL value of its own setting past COMMIT.
+ add(`BEGIN;
+SET LOCAL ROLE s;
+SET LOCAL search_path = j;
+SET search_path FROM CURRENT;
+COMMIT;`)
+ require.Equal(t, []string{"SET search_path = a", "SET ROLE r", "SET search_path = d", "RESET search_path", "SET search_path = h", "SET LOCAL search_path = j", "SET search_path FROM CURRENT"}, settings.statements())
+}
diff --git a/backend/plugin/advisor/utils.go b/backend/plugin/advisor/utils.go
index e8fa271329..d8fce879c2 100644
--- a/backend/plugin/advisor/utils.go
+++ b/backend/plugin/advisor/utils.go
@@ -9,11 +9,50 @@ import (
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
+// ExplainBudget caps the statements a row limit rule EXPLAINs at common.MaximumLintExplainSize and
+// counts the statements it leaves unchecked.
+type ExplainBudget struct {
+ used int
+ skipped int
+ firstSkipped *storepb.Position
+}
+
+// Spend reports whether the statement at position may be explained, recording it as skipped
+// otherwise.
+func (b *ExplainBudget) Spend(position *storepb.Position) bool {
+ if b.used < common.MaximumLintExplainSize {
+ b.used++
+ return true
+ }
+ if b.skipped == 0 {
+ b.firstSkipped = position
+ }
+ b.skipped++
+ return false
+}
+
+// AppendSkippedAdvice appends one warning covering the skipped statements, if any. It is a warning
+// whatever the rule's level.
+func (b *ExplainBudget) AppendSkippedAdvice(adviceList []*storepb.Advice, title string, adviceCode code.Code) []*storepb.Advice {
+ if b.skipped == 0 {
+ return adviceList
+ }
+ return append(adviceList, &storepb.Advice{
+ Status: storepb.Advice_WARNING,
+ Code: adviceCode.Int32(),
+ Title: title,
+ Content: fmt.Sprintf("Only the first %d statements were estimated; %d more were not checked against the row limit.", common.MaximumLintExplainSize, b.skipped),
+ StartPosition: b.firstSkipped,
+ })
+}
+
type QueryContext struct {
TenantMode bool
PreExecutions []string
@@ -152,7 +191,8 @@ func ContainsDDL(engine storepb.Engine, parsedStatements []base.ParsedStatement)
case storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED,
storepb.StatementType_INSERT,
storepb.StatementType_UPDATE,
- storepb.StatementType_DELETE:
+ storepb.StatementType_DELETE,
+ storepb.StatementType_MERGE:
default:
return true
}
diff --git a/backend/plugin/db/cockroachdb/stats.go b/backend/plugin/db/cockroachdb/stats.go
index b4e15c0402..93766fdc42 100644
--- a/backend/plugin/db/cockroachdb/stats.go
+++ b/backend/plugin/db/cockroachdb/stats.go
@@ -4,62 +4,223 @@ import (
"context"
"database/sql"
"fmt"
+ "math"
"regexp"
+ "slices"
"strconv"
+ "strings"
+ "unicode/utf8"
"github.com/cockroachdb/cockroach-go/v2/crdb"
"github.com/pkg/errors"
-)
-var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+// CountAffectedRows returns the optimizer's estimate of the rows the statement modifies. A search
+// path that base.WithSearchPath put before the statement is set for the EXPLAIN.
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)
- var rowCount int64
- if err := crdb.Execute(func() error {
- rows, err := d.db.QueryContext(ctx, explainSQL)
+ setup, statement := base.SplitSearchPath(statement)
+ var settings []string
+ if setup != "" {
+ settings = append(settings, setup)
+ }
+ plan, err := d.explain(ctx, statement, settings...)
+ if err != nil {
+ return 0, err
+ }
+ rows, err := getAffectedRowsFromPlan(plan)
+ if err == nil || !hasDeleteRangeNode(plan) {
+ return rows, err
+ }
+ // A delete range node has no row estimate, while the plan of the same DELETE without the fast
+ // path has. Versions without the setting keep the original error.
+ fallbackPlan, fallbackErr := d.explain(ctx, statement, append(settings, "SET LOCAL optimizer_use_delete_range_fast_path = off")...)
+ if fallbackErr != nil {
+ return 0, err
+ }
+ return getAffectedRowsFromPlan(fallbackPlan)
+}
+
+// explain returns the EXPLAIN output of the statement after the SET LOCAL settings, which run in a
+// transaction that is rolled back so the pooled connection keeps its session.
+func (d *Driver) explain(ctx context.Context, statement string, settings ...string) ([]string, error) {
+ var plan []string
+ err := crdb.Execute(func() error {
+ if len(settings) == 0 {
+ var err error
+ plan, err = queryPlan(ctx, d.db.QueryContext, statement)
+ return err
+ }
+ tx, err := d.db.BeginTx(ctx, nil)
if err != nil {
return err
}
- defer rows.Close()
-
- // test-bb=# EXPLAIN INSERT INTO t SELECT * FROM t;
- // QUERY PLAN
- // -------------------------------------------------------------
- // Insert on t (cost=0.00..1.03 rows=0 width=0)
- // -> Seq Scan on t t_1 (cost=0.00..1.03 rows=3 width=520)
- // (2 rows)
- //
- // d1=# explain select * from h1;
- // QUERY PLAN
- // ------------------------------------------------------
- // Seq Scan on h1 (cost=0.00..35.50 rows=2550 width=4)
- // (1 row)
- for rows.Next() {
- var planRow sql.NullString
- if err := rows.Scan(&planRow); err != nil {
+ defer tx.Rollback()
+ for _, setting := range settings {
+ if _, err := tx.ExecContext(ctx, setting); err != nil {
return err
}
+ }
+ plan, err = queryPlan(ctx, tx.QueryContext, statement)
+ return err
+ })
+ return plan, err
+}
- if !planRow.Valid {
- continue
- }
+func queryPlan(ctx context.Context, query func(context.Context, string, ...any) (*sql.Rows, error), statement string) ([]string, error) {
+ rows, err := query(ctx, fmt.Sprintf("EXPLAIN %s", statement))
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ var plan []string
+ for rows.Next() {
+ var line sql.NullString
+ if err := rows.Scan(&line); err != nil {
+ return nil, err
+ }
+ plan = append(plan, line.String)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return plan, nil
+}
- matches := rowsRegexp.FindStringSubmatch(planRow.String)
- if len(matches) != 2 {
- continue
- }
- v, err := strconv.ParseInt(matches[1], 10, 64)
- if err != nil {
- return errors.Errorf("failed to get integer from %q", matches[1])
+type planNode struct {
+ name string
+ // column is the rune offset of the node's bullet, which grows with the node's depth.
+ column int
+ attributes []string
+}
+
+// parsePlanNodes returns the nodes of EXPLAIN text output in pre-order. A node line has a bullet
+// after the tree drawing, and the lines before the next node line are the node's attributes.
+func parsePlanNodes(plan []string) []planNode {
+ var nodes []planNode
+ for _, line := range plan {
+ line = strings.TrimRight(line, " \t\r")
+ text := strings.TrimLeft(line, " │├└─")
+ if name, ok := strings.CutPrefix(text, "• "); ok {
+ nodes = append(nodes, planNode{
+ name: name,
+ column: utf8.RuneCountInString(line[:len(line)-len(text)]),
+ })
+ continue
+ }
+ if len(nodes) > 0 && text != "" {
+ nodes[len(nodes)-1].attributes = append(nodes[len(nodes)-1].attributes, text)
+ }
+ }
+ return nodes
+}
+
+func hasDeleteRangeNode(plan []string) bool {
+ return slices.ContainsFunc(parsePlanNodes(plan), func(node planNode) bool {
+ return node.name == "delete range"
+ })
+}
+
+var mutationNodeNames = map[string]bool{
+ "insert": true,
+ "insert fast path": true,
+ "upsert": true,
+ "update": true,
+ "delete": true,
+ "delete range": true,
+}
+
+// getAffectedRowsFromPlan sums the rows written by the mutation nodes in EXPLAIN text output,
+// including the mutations of CTEs and statement sources. Foreign key cascades are left out, as
+// CockroachDB leaves them out of the rows a statement affects. It fails when the plan has no
+// mutation node or a mutation node has no row count.
+func getAffectedRowsFromPlan(plan []string) (int64, error) {
+ nodes := parsePlanNodes(plan)
+ var total float64
+ mutationCount := 0
+ cascadeColumn := -1
+ for i, node := range nodes {
+ if cascadeColumn >= 0 && node.column > cascadeColumn {
+ continue
+ }
+ cascadeColumn = -1
+ if node.name == "fk-cascade" {
+ cascadeColumn = node.column
+ continue
+ }
+ if !mutationNodeNames[node.name] {
+ continue
+ }
+ rows, ok := getMutationRows(nodes, i)
+ if !ok {
+ return 0, errors.Errorf("the %s node in the plan has no row count estimate", node.name)
+ }
+ total += rows
+ mutationCount++
+ }
+ if mutationCount == 0 {
+ return 0, errors.New("the plan has no mutation node")
+ }
+ return common.RoundRows(total), nil
+}
+
+// getMutationRows returns the rows written by the mutation node nodes[i]. A mutation that returns
+// rows has its own estimate. Otherwise the first row count in its subtree, in pre-order, estimates
+// its input, capped by the counts of the limit nodes above it, which have no estimate of their own.
+func getMutationRows(nodes []planNode, i int) (float64, bool) {
+ limit := math.Inf(1)
+ for j := i; j < len(nodes); j++ {
+ node := nodes[j]
+ if j > i && node.column <= nodes[i].column {
+ break
+ }
+ for _, attribute := range node.attributes {
+ key, value, _ := strings.Cut(attribute, ": ")
+ switch {
+ case key == "estimated row count":
+ rows, ok := parseEstimatedRowCount(value)
+ return math.Min(rows, limit), ok
+ case key == "size" && (node.name == "values" || node.name == "insert fast path"):
+ rows, ok := parseValuesSize(value)
+ return math.Min(rows, limit), ok
+ case key == "count" && node.name == "limit":
+ if count, ok := parseCount(value); ok {
+ limit = math.Min(limit, count)
+ }
+ default:
}
- rowCount = v
}
- err = rows.Err()
- return err
- }); err != nil {
- return 0, err
}
+ return 0, false
+}
- return rowCount, nil
+// parseEstimatedRowCount parses "1,000 (10% of the table; stats collected 1 minute ago)". A node
+// under a soft limit shows a "min - max" range, whose maximum leaves the limit to the nodes above.
+func parseEstimatedRowCount(value string) (float64, bool) {
+ value, _, _ = strings.Cut(value, " (")
+ if _, maximum, ok := strings.Cut(value, " - "); ok {
+ value = maximum
+ }
+ return parseCount(value)
+}
+
+var valuesSizeRegexp = regexp.MustCompile(`^[\d,]+ columns?, ([\d,]+) rows?$`)
+
+// parseValuesSize parses the rows of "5 columns, 2 rows".
+func parseValuesSize(value string) (float64, bool) {
+ match := valuesSizeRegexp.FindStringSubmatch(value)
+ if match == nil {
+ return 0, false
+ }
+ return parseCount(match[1])
+}
+
+func parseCount(value string) (float64, bool) {
+ count, err := strconv.ParseUint(strings.ReplaceAll(value, ",", ""), 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return float64(count), true
}
diff --git a/backend/plugin/db/cockroachdb/stats_test.go b/backend/plugin/db/cockroachdb/stats_test.go
new file mode 100644
index 0000000000..78eea0f223
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/stats_test.go
@@ -0,0 +1,232 @@
+package cockroachdb
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// The fixtures are EXPLAIN outputs captured through this driver from CockroachDB v25.2. After ANALYZE,
+// t has 1000 rows with an index on c = id % 100 and none on d = id % 7, s has 100 rows (10 flagged),
+// big has 100 rows per s id, big2 has 10000 rows and no secondary index, and parent has 100 rows with
+// 10 cascading child rows each. t2 is empty and has no statistics.
+func TestGetAffectedRowsFromPlan(t *testing.T) {
+ for _, tc := range []struct {
+ fixture string
+ want int64
+ }{
+ {fixture: "update_index_join.txt", want: 100},
+ // The filter estimates 143 of the 1,000 scanned rows.
+ {fixture: "update_filter.txt", want: 143},
+ // The limit caps the filter's 143 rows.
+ {fixture: "delete_filter_limit.txt", want: 10},
+ {fixture: "update_from_lookup_join.txt", want: 1000},
+ {fixture: "update_exists_hash_semi_join.txt", want: 1000},
+ // The subquery in SET is a sibling of the UPDATE, not its input.
+ {fixture: "update_subquery_in_set.txt", want: 100},
+ // The anti join against t2 has no estimate, so its input's estimate is used.
+ {fixture: "insert_select_on_conflict_do_nothing.txt", want: 100},
+ // The foreign key check after the INSERT is not its input.
+ {fixture: "insert_select_fk_check.txt", want: 49},
+ {fixture: "upsert_select.txt", want: 100},
+ {fixture: "upsert_values_select.txt", want: 1},
+ {fixture: "insert_fast_path.txt", want: 2},
+ // The DELETE returns rows to the CTE, so it has an estimate of its own.
+ {fixture: "cte_delete_select.txt", want: 5000},
+ // The statement deletes 100 rows and inserts them into t2.
+ {fixture: "insert_select_statement_source_delete.txt", want: 200},
+ {fixture: "delete_range_fast_path_off.txt", want: 100},
+ // The DELETE removes 10 parent rows; the 100 child rows it cascades to are not counted.
+ {fixture: "delete_fk_cascade_fast_path_off.txt", want: 10},
+ } {
+ t.Run(tc.fixture, func(t *testing.T) {
+ rows, err := getAffectedRowsFromPlan(readPlanFixture(t, tc.fixture))
+ require.NoError(t, err)
+ require.Equal(t, tc.want, rows)
+ })
+ }
+}
+
+func TestGetAffectedRowsFromPlanError(t *testing.T) {
+ for name, plan := range map[string][]string{
+ "select": readPlanFixture(t, "select.txt"),
+ "delete range": readPlanFixture(t, "delete_range.txt"),
+ "missing stats": readPlanFixture(t, "update_missing_stats.txt"),
+ "empty": nil,
+ "unparsable estimate": {"• update", "│", "└── • scan", " estimated row count: many"},
+ "limit count without an estimate": {"• delete", "│", "└── • limit", " count: 10"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ rows, err := getAffectedRowsFromPlan(plan)
+ require.Error(t, err)
+ require.Zero(t, rows)
+ })
+ }
+}
+
+func TestCountAffectedRows(t *testing.T) {
+ const statement = "DELETE FROM big2 WHERE id <= 100"
+ const setLocal = "SET LOCAL optimizer_use_delete_range_fast_path = off"
+ for _, tc := range []struct {
+ name string
+ // input is the statement passed to CountAffectedRows, when it is not statement.
+ input string
+ plans [][]string
+ setLocalErr error
+ want int64
+ wantErr string
+ wantStatements []string
+ }{
+ {
+ name: "search_path_is_set_for_the_explain",
+ input: base.WithSearchPath(statement, []string{"app"}),
+ plans: [][]string{readPlanFixture(t, "delete_range_fast_path_off.txt")},
+ want: 100,
+ wantStatements: []string{"BEGIN", `SET LOCAL search_path TO "app"`, "EXPLAIN " + statement, "ROLLBACK"},
+ },
+ {
+ name: "plan_with_an_estimate",
+ plans: [][]string{readPlanFixture(t, "delete_range_fast_path_off.txt")},
+ want: 100,
+ wantStatements: []string{"EXPLAIN " + statement},
+ },
+ {
+ name: "delete_range_is_planned_again_without_the_fast_path",
+ plans: [][]string{readPlanFixture(t, "delete_range.txt"), readPlanFixture(t, "delete_range_fast_path_off.txt")},
+ want: 100,
+ wantStatements: []string{"EXPLAIN " + statement, "BEGIN", setLocal, "EXPLAIN " + statement, "ROLLBACK"},
+ },
+ {
+ name: "version_without_the_setting_keeps_the_delete_range_error",
+ plans: [][]string{readPlanFixture(t, "delete_range.txt")},
+ setLocalErr: errors.New(`unrecognized configuration parameter "optimizer_use_delete_range_fast_path"`),
+ wantErr: "the delete range node in the plan has no row count estimate",
+ wantStatements: []string{"EXPLAIN " + statement, "BEGIN", setLocal, "ROLLBACK"},
+ },
+ {
+ name: "plan_without_delete_range_is_not_planned_again",
+ plans: [][]string{readPlanFixture(t, "select.txt")},
+ wantErr: "the plan has no mutation node",
+ wantStatements: []string{"EXPLAIN " + statement},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := &fakeExplainConnector{plans: tc.plans, setLocalErr: tc.setLocalErr}
+ db := sql.OpenDB(connector)
+ defer db.Close()
+
+ input := tc.input
+ if input == "" {
+ input = statement
+ }
+ got, err := (&Driver{db: db}).CountAffectedRows(context.Background(), input)
+ if tc.wantErr != "" {
+ require.EqualError(t, err, tc.wantErr)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ }
+ require.Equal(t, tc.wantStatements, connector.statements)
+ })
+ }
+}
+
+func readPlanFixture(t *testing.T, name string) []string {
+ t.Helper()
+ content, err := os.ReadFile(filepath.Join("test-data", "explain-plan", name))
+ require.NoError(t, err)
+ return strings.Split(strings.TrimSuffix(string(content), "\n"), "\n")
+}
+
+// fakeExplainConnector answers each EXPLAIN with the next plan and records every statement,
+// including transaction control.
+type fakeExplainConnector struct {
+ plans [][]string
+ setLocalErr error
+ statements []string
+}
+
+func (c *fakeExplainConnector) Connect(context.Context) (driver.Conn, error) {
+ return &fakeExplainConn{connector: c}, nil
+}
+
+func (*fakeExplainConnector) Driver() driver.Driver {
+ return nil
+}
+
+type fakeExplainConn struct {
+ connector *fakeExplainConnector
+}
+
+func (*fakeExplainConn) Prepare(string) (driver.Stmt, error) {
+ return nil, driver.ErrSkip
+}
+
+func (*fakeExplainConn) Close() error {
+ return nil
+}
+
+func (c *fakeExplainConn) Begin() (driver.Tx, error) {
+ c.connector.statements = append(c.connector.statements, "BEGIN")
+ return &fakeExplainTx{connector: c.connector}, nil
+}
+
+func (c *fakeExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {
+ c.connector.statements = append(c.connector.statements, query)
+ return driver.ResultNoRows, c.connector.setLocalErr
+}
+
+func (c *fakeExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+ c.connector.statements = append(c.connector.statements, query)
+ if len(c.connector.plans) == 0 {
+ return nil, errors.New("unexpected EXPLAIN")
+ }
+ plan := c.connector.plans[0]
+ c.connector.plans = c.connector.plans[1:]
+ return &fakeExplainRows{lines: plan}, nil
+}
+
+type fakeExplainTx struct {
+ connector *fakeExplainConnector
+}
+
+func (tx *fakeExplainTx) Commit() error {
+ tx.connector.statements = append(tx.connector.statements, "COMMIT")
+ return nil
+}
+
+func (tx *fakeExplainTx) Rollback() error {
+ tx.connector.statements = append(tx.connector.statements, "ROLLBACK")
+ return nil
+}
+
+type fakeExplainRows struct {
+ lines []string
+}
+
+func (*fakeExplainRows) Columns() []string {
+ return []string{"info"}
+}
+
+func (*fakeExplainRows) Close() error {
+ return nil
+}
+
+func (r *fakeExplainRows) Next(dest []driver.Value) error {
+ if len(r.lines) == 0 {
+ return io.EOF
+ }
+ dest[0] = r.lines[0]
+ r.lines = r.lines[1:]
+ return nil
+}
diff --git a/backend/plugin/db/cockroachdb/sync.go b/backend/plugin/db/cockroachdb/sync.go
index 9888136eef..4140c35286 100644
--- a/backend/plugin/db/cockroachdb/sync.go
+++ b/backend/plugin/db/cockroachdb/sync.go
@@ -78,6 +78,9 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
if databaseMetadata == nil {
return nil, common.Errorf(common.NotFound, "database %q not found", d.databaseName)
}
+ if err := d.db.QueryRowContext(ctx, "SHOW search_path").Scan(&databaseMetadata.SearchPath); err != nil {
+ return nil, errors.Wrapf(err, "failed to get search path for database %q", d.databaseName)
+ }
isAtLeastPG10 := isAtLeastPG10(d.connectionCtx.EngineVersion)
txn, err := d.db.BeginTx(ctx, nil)
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txt
new file mode 100644
index 0000000000..fd5f021289
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/cte_delete_select.txt
@@ -0,0 +1,29 @@
+distribution: local
+vectorized: true
+
+• root
+│
+├── • group (scalar)
+│ │ estimated row count: 1
+│ │
+│ └── • scan buffer
+│ estimated row count: 5,000
+│ label: buffer 1 (d)
+│
+└── • subquery
+ │ id: @S1
+ │ original sql: DELETE FROM big WHERE s_id <= 50 RETURNING id
+ │ exec mode: discard all rows
+ │
+ └── • buffer
+ │ label: buffer 1 (d)
+ │
+ └── • delete
+ │ estimated row count: 5,000
+ │ from: big
+ │
+ └── • scan
+ estimated row count: 5,000 (50% of the table; stats collected 40 seconds ago)
+ table: big@idx_s
+ spans: [ - /50]
+ locking strength: for update
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txt
new file mode 100644
index 0000000000..edcca5af10
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_filter_limit.txt
@@ -0,0 +1,22 @@
+distribution: local
+vectorized: true
+
+• delete
+│ from: t
+│ auto commit
+│
+└── • limit
+ │ count: 10
+ │
+ └── • filter
+ │ estimated row count: 143
+ │ filter: d = 3
+ │
+ └── • scan
+ estimated row count: 70 - 1,000 (100% of the table; stats collected 40 seconds ago)
+ table: t@t_pkey
+ spans: FULL SCAN (SOFT LIMIT)
+
+index recommendations: 1
+1. type: index creation
+ SQL command: CREATE INDEX ON capture.public.t (d) STORING (c);
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txt
new file mode 100644
index 0000000000..90bad021ff
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_fk_cascade_fast_path_off.txt
@@ -0,0 +1,28 @@
+distribution: local
+vectorized: true
+
+• root
+│
+├── • delete
+│ │ from: parent
+│ │
+│ └── • scan
+│ estimated row count: 10 (10% of the table; stats collected 39 seconds ago)
+│ table: parent@parent_pkey
+│ spans: [ - /10]
+│ locking strength: for update
+│
+└── • fk-cascade
+ │ fk: child_parent_id_fkey
+ │
+ └── • delete
+ │ from: child
+ │
+ └── • filter
+ │ estimated row count: 100
+ │ filter: parent_id <= 10
+ │
+ └── • scan
+ estimated row count: 1,000 (100% of the table; stats collected 39 seconds ago)
+ table: child@child_pkey
+ spans: FULL SCAN
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txt
new file mode 100644
index 0000000000..2d2382b49c
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range.txt
@@ -0,0 +1,6 @@
+distribution: local
+vectorized: true
+
+• delete range
+ from: big2
+ spans: [ - /100]
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txt
new file mode 100644
index 0000000000..f6c7baf4ca
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/delete_range_fast_path_off.txt
@@ -0,0 +1,11 @@
+distribution: local
+vectorized: true
+
+• delete
+│ from: big2
+│
+└── • scan
+ estimated row count: 100 (1.0% of the table; stats collected 40 seconds ago)
+ table: big2@big2_pkey
+ spans: [ - /100]
+ locking strength: for update
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txt
new file mode 100644
index 0000000000..15cd23e67b
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_fast_path.txt
@@ -0,0 +1,7 @@
+distribution: local
+vectorized: true
+
+• insert fast path
+ into: t2(id, c, d, grp, v)
+ auto commit
+ size: 5 columns, 2 rows
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txt
new file mode 100644
index 0000000000..ab8c249c1e
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_fk_check.txt
@@ -0,0 +1,31 @@
+distribution: local
+vectorized: true
+
+• root
+│
+├── • insert
+│ │ into: child(id, parent_id, v)
+│ │
+│ └── • buffer
+│ │ label: buffer 1
+│ │
+│ └── • render
+│ │
+│ └── • scan
+│ estimated row count: 49 (49% of the table; stats collected 39 seconds ago)
+│ table: parent@parent_pkey
+│ spans: [ - /49]
+│
+└── • constraint-check
+ │
+ └── • error if rows
+ │
+ └── • lookup join (anti)
+ │ estimated row count: 0
+ │ table: parent@parent_pkey
+ │ equality: (id) = (id)
+ │ equality cols are key
+ │
+ └── • scan buffer
+ estimated row count: 49
+ label: buffer 1
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txt
new file mode 100644
index 0000000000..e2de9b7396
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_on_conflict_do_nothing.txt
@@ -0,0 +1,25 @@
+distribution: local
+vectorized: true
+
+• insert
+│ into: t2(id, c, d, grp, v)
+│ auto commit
+│ arbiter indexes: t2_pkey
+│
+└── • lookup join (anti)
+ │ table: t2@t2_pkey
+ │ equality: (id) = (id)
+ │ equality cols are key
+ │
+ └── • index join
+ │ estimated row count: 100
+ │ table: t@t_pkey
+ │
+ └── • scan
+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+ table: t@idx_c
+ spans: [ - /9]
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txt
new file mode 100644
index 0000000000..6850426752
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/insert_select_statement_source_delete.txt
@@ -0,0 +1,38 @@
+distribution: local
+vectorized: true
+
+• root
+│
+├── • insert
+│ │ into: t2(id, c, d, grp, v)
+│ │
+│ └── • scan buffer
+│ estimated row count: 100
+│ label: buffer 1
+│
+└── • subquery
+ │ id: @S1
+ │ original sql: DELETE FROM t WHERE c < 10 RETURNING *
+ │ exec mode: discard all rows
+ │
+ └── • buffer
+ │ label: buffer 1
+ │
+ └── • delete
+ │ estimated row count: 100
+ │ from: t
+ │
+ └── • index join
+ │ estimated row count: 100
+ │ table: t@t_pkey
+ │ locking strength: for update
+ │
+ └── • scan
+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+ table: t@idx_c
+ spans: [ - /9]
+ locking strength: for update
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txt
new file mode 100644
index 0000000000..a9f68cb42c
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/select.txt
@@ -0,0 +1,15 @@
+distribution: local
+vectorized: true
+
+• index join
+│ estimated row count: 100
+│ table: t@t_pkey
+│
+└── • scan
+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+ table: t@idx_c
+ spans: [ - /9]
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txt
new file mode 100644
index 0000000000..17cff77bea
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_exists_hash_semi_join.txt
@@ -0,0 +1,32 @@
+distribution: local
+vectorized: true
+
+• update
+│ table: t
+│ set: v
+│ auto commit
+│
+└── • render
+ │
+ └── • hash join (semi)
+ │ estimated row count: 1,000
+ │ equality: (grp) = (id)
+ │ right cols are key
+ │
+ ├── • scan
+ │ estimated row count: 1,000 (100% of the table; stats collected 40 seconds ago)
+ │ table: t@t_pkey
+ │ spans: FULL SCAN
+ │
+ └── • filter
+ │ estimated row count: 10
+ │ filter: flag
+ │
+ └── • scan
+ estimated row count: 100 (100% of the table; stats collected 40 seconds ago)
+ table: s@s_pkey
+ spans: FULL SCAN
+
+index recommendations: 1
+1. type: index creation
+ SQL command: CREATE INDEX ON capture.public.t (grp) STORING (c, d, v);
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txt
new file mode 100644
index 0000000000..f2dbdd922c
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_filter.txt
@@ -0,0 +1,22 @@
+distribution: local
+vectorized: true
+
+• update
+│ table: t
+│ set: v
+│ auto commit
+│
+└── • render
+ │
+ └── • filter
+ │ estimated row count: 143
+ │ filter: d = 3
+ │
+ └── • scan
+ estimated row count: 1,000 (100% of the table; stats collected 40 seconds ago)
+ table: t@t_pkey
+ spans: FULL SCAN
+
+index recommendations: 1
+1. type: index creation
+ SQL command: CREATE INDEX ON capture.public.t (d) STORING (c, grp, v);
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txt
new file mode 100644
index 0000000000..d5735708b5
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_from_lookup_join.txt
@@ -0,0 +1,33 @@
+distribution: local
+vectorized: true
+
+• update
+│ table: big
+│ set: v
+│ auto commit
+│
+└── • render
+ │
+ └── • lookup join
+ │ estimated row count: 1,000
+ │ table: big@big_pkey
+ │ equality: (id) = (id)
+ │ equality cols are key
+ │
+ └── • lookup join
+ │ estimated row count: 1,000
+ │ table: big@idx_s
+ │ equality: (id) = (s_id)
+ │
+ └── • filter
+ │ estimated row count: 10
+ │ filter: flag
+ │
+ └── • scan
+ estimated row count: 100 (100% of the table; stats collected 40 seconds ago)
+ table: s@s_pkey
+ spans: FULL SCAN
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.big (s_id) STORING (v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.big@idx_s NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.big@idx_s` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txt
new file mode 100644
index 0000000000..3859871c15
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_index_join.txt
@@ -0,0 +1,24 @@
+distribution: local
+vectorized: true
+
+• update
+│ table: t
+│ set: v
+│ auto commit
+│
+└── • render
+ │
+ └── • index join
+ │ estimated row count: 100
+ │ table: t@t_pkey
+ │ locking strength: for update
+ │
+ └── • scan
+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+ table: t@idx_c
+ spans: [ - /9]
+ locking strength: for update
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txt
new file mode 100644
index 0000000000..0ce7fcbc26
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_missing_stats.txt
@@ -0,0 +1,21 @@
+distribution: local
+vectorized: true
+
+• update
+│ table: t2
+│ set: v
+│ auto commit
+│
+└── • render
+ │
+ └── • filter
+ │ filter: c = 1
+ │
+ └── • scan
+ missing stats
+ table: t2@t2_pkey
+ spans: FULL SCAN
+
+index recommendations: 1
+1. type: index creation
+ SQL command: CREATE INDEX ON capture.public.t2 (c) STORING (d, grp, v);
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txt
new file mode 100644
index 0000000000..1a7937fc42
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/update_subquery_in_set.txt
@@ -0,0 +1,45 @@
+distribution: local
+vectorized: true
+
+• root
+│
+├── • update
+│ │ table: t
+│ │ set: v
+│ │ auto commit
+│ │
+│ └── • render
+│ │
+│ └── • index join
+│ │ estimated row count: 100
+│ │ table: t@t_pkey
+│ │ locking strength: for update
+│ │
+│ └── • scan
+│ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+│ table: t@idx_c
+│ spans: [ - /9]
+│ locking strength: for update
+│
+└── • subquery
+ │ id: @S1
+ │ original sql: (SELECT v FROM s JOIN t ON t.id = s.id LIMIT 1)
+ │ exec mode: one row
+ │
+ └── • limit
+ │ count: 1
+ │
+ └── • lookup join
+ │ estimated row count: 100
+ │ table: s@s_pkey
+ │ equality: (id) = (id)
+ │ equality cols are key
+ │
+ └── • scan
+ estimated row count: 100 - 1,000 (100% of the table; stats collected 40 seconds ago)
+ table: t@t_pkey
+ spans: FULL SCAN (SOFT LIMIT)
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txt
new file mode 100644
index 0000000000..c8a528438e
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_select.txt
@@ -0,0 +1,19 @@
+distribution: local
+vectorized: true
+
+• upsert
+│ into: t2(id, c, d, grp, v)
+│ auto commit
+│
+└── • index join
+ │ estimated row count: 100
+ │ table: t@t_pkey
+ │
+ └── • scan
+ estimated row count: 100 (10% of the table; stats collected 40 seconds ago)
+ table: t@idx_c
+ spans: [ - /9]
+
+index recommendations: 1
+1. type: index replacement
+ SQL commands: CREATE INDEX ON capture.public.t (c) STORING (d, grp, v) /* After successfully creating the replacement index, manually run: `ALTER INDEX capture.public.t@idx_c NOT VISIBLE` and then, after verifying workload performance, manually run: `DROP INDEX capture.public.t@idx_c` */;
diff --git a/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txt b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txt
new file mode 100644
index 0000000000..235e3b5ad9
--- /dev/null
+++ b/backend/plugin/db/cockroachdb/test-data/explain-plan/upsert_values_select.txt
@@ -0,0 +1,9 @@
+distribution: local
+vectorized: true
+
+• upsert
+│ into: t2(id, c, d, grp, v)
+│ auto commit
+│
+└── • values
+ size: 5 columns, 1 row
diff --git a/backend/plugin/db/mssql/stats.go b/backend/plugin/db/mssql/stats.go
index bfe1653c07..8faf4b7ed1 100644
--- a/backend/plugin/db/mssql/stats.go
+++ b/backend/plugin/db/mssql/stats.go
@@ -3,8 +3,8 @@ package mssql
import (
"context"
"database/sql"
- "math"
+ "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/plugin/db/util"
)
@@ -47,7 +47,7 @@ func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64
}
if rowsColumn.Valid {
- return int64(math.Round(rowsColumn.Float64)), nil
+ return common.RoundRows(rowsColumn.Float64), nil
}
}
if err := rows.Err(); err != nil {
diff --git a/backend/plugin/db/mssql/stats_test.go b/backend/plugin/db/mssql/stats_test.go
index 8f695eb5f5..35c287e7cc 100644
--- a/backend/plugin/db/mssql/stats_test.go
+++ b/backend/plugin/db/mssql/stats_test.go
@@ -32,10 +32,27 @@ GO
INSERT INTO dbo.t (c1) VALUES (1),(2),(3),(4),(5);
GO
+CREATE TABLE dbo.s (id INT PRIMARY KEY, flag BIT NOT NULL);
+CREATE TABLE dbo.big (id INT PRIMARY KEY, s_id INT NOT NULL, v INT NOT NULL);
+CREATE TABLE dbo.big_archive (id INT PRIMARY KEY, s_id INT NOT NULL, v INT NOT NULL);
+GO
+
+-- s flags 10 of its 100 rows and big holds 100 rows for each row of s, so
+-- joining big to the flagged rows matches 1000 of its 10000 rows.
+INSERT INTO dbo.s SELECT n, CASE WHEN n <= 10 THEN 1 ELSE 0 END FROM (SELECT TOP (100) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n FROM sys.all_objects) AS seq;
+INSERT INTO dbo.big SELECT n, n % 100 + 1, 0 FROM (SELECT TOP (10000) ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n FROM sys.all_objects AS a CROSS JOIN sys.all_objects AS b) AS seq;
+GO
+
+CREATE INDEX big_s_id ON dbo.big (s_id);
+CREATE STATISTICS s_flag ON dbo.s (flag);
+GO
+
-- EstimateRows is a cardinality estimate, so it is only stable once the
-- optimizer has statistics. Without them SQL Server falls back to guessing a
-- fixed selectivity for a range predicate and the counts below drift.
UPDATE STATISTICS dbo.t WITH FULLSCAN;
+UPDATE STATISTICS dbo.s WITH FULLSCAN;
+UPDATE STATISTICS dbo.big WITH FULLSCAN;
GO
`)
@@ -68,6 +85,63 @@ GO
require.EqualValues(t, 3, rows)
})
+ for _, tc := range []struct {
+ name string
+ statement string
+ // changedRows is how many rows the statement would change. An estimate can miss it,
+ // as MERGE estimates 899 of 1000 on SQL Server 2022, so the bounds allow a factor of
+ // two, which still excludes the 10, 100, and 10000 rows that the join inputs estimate.
+ changedRows int64
+ }{
+ {
+ name: "delete_with_predicate",
+ statement: `DELETE FROM dbo.big WHERE s_id <= 3;`,
+ changedRows: 300,
+ },
+ {
+ name: "delete_join",
+ statement: `DELETE b FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,
+ changedRows: 1000,
+ },
+ {
+ name: "update_from_join",
+ statement: `UPDATE b SET v = 1 FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,
+ changedRows: 1000,
+ },
+ {
+ name: "delete_where_in_subquery",
+ statement: `DELETE FROM dbo.big WHERE s_id IN (SELECT id FROM dbo.s WHERE flag = 1);`,
+ changedRows: 1000,
+ },
+ {
+ name: "insert_select_with_filter",
+ statement: `INSERT INTO dbo.big_archive SELECT id, s_id, v FROM dbo.big WHERE s_id > 95;`,
+ changedRows: 500,
+ },
+ {
+ name: "insert_select_with_join",
+ statement: `INSERT INTO dbo.big_archive SELECT b.id, b.s_id, b.v FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1;`,
+ changedRows: 1000,
+ },
+ {
+ name: "merge",
+ statement: `MERGE dbo.big_archive AS tgt
+USING (SELECT b.id, b.s_id, b.v FROM dbo.big AS b JOIN dbo.s AS s ON b.s_id = s.id WHERE s.flag = 1) AS src
+ON tgt.id = src.id
+WHEN NOT MATCHED THEN INSERT (id, s_id, v) VALUES (src.id, src.s_id, src.v);`,
+ changedRows: 1000,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ rows, err := driver.CountAffectedRows(ctx, tc.statement)
+ require.NoError(t, err)
+ require.Greater(t, rows, tc.changedRows/2)
+ require.LessOrEqual(t, rows, tc.changedRows*2)
+ })
+ }
+
// The seam between the two halves: neither the tsql extractor tests nor the
// subtests above can catch a sample that the extractor emits but SHOWPLAN
// rejects, so feed the extractor's own output in rather than a literal.
@@ -82,9 +156,9 @@ GO
summary, err := parserbase.ExtractChangedResources(
storepb.Engine_MSSQL, "resource_change_test", "dbo", nil, parserbase.ExtractASTs(stmts), statement)
require.NoError(t, err)
- require.Len(t, summary.SampleDMLS, 1)
+ require.Len(t, summary.DMLStatements, 1)
- sampled, err := driver.CountAffectedRows(ctx, summary.SampleDMLS[0])
+ sampled, err := driver.CountAffectedRows(ctx, summary.DMLStatements[0])
require.NoError(t, err, "SHOWPLAN must accept the sample the extractor emits")
want, err := driver.CountAffectedRows(ctx, bare)
require.NoError(t, err)
diff --git a/backend/plugin/db/mysql/stats.go b/backend/plugin/db/mysql/stats.go
index afe6db5da7..6835dca530 100644
--- a/backend/plugin/db/mysql/stats.go
+++ b/backend/plugin/db/mysql/stats.go
@@ -3,221 +3,165 @@ package mysql
import (
"context"
"database/sql"
- "encoding/json"
- "fmt"
+ "log/slog"
+ "slices"
+ "strconv"
"strings"
"github.com/bytebase/omni/mysql/ast"
+ "github.com/go-sql-driver/mysql"
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/db/util"
mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
)
+// erUnknownSystemVariable is the MySQL error number for setting a variable the server lacks.
+const erUnknownSystemVariable = 1193
+
+// CountAffectedRows returns the planner's estimate of the rows the INSERT, UPDATE, or DELETE
+// statement modifies.
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
if d.dbType == storepb.Engine_OCEANBASE {
- return countAffectedRowsForOceanBase(ctx, d.db, statement)
- }
-
- // Prefer the JSON query plan: the tabular EXPLAIN's first "rows" value may be the
- // scan estimate of a driving table unrelated to the DML target (BYT-9858). Fall
- // back to the tabular heuristic when no flagged target node is found (e.g. INSERT
- // plans, or servers that don't mark the target).
- if count, ok := d.countAffectedRowsFromJSONPlan(ctx, statement); ok {
- return capAffectedRowsByLimit(count, statement), nil
+ rows, err := d.db.QueryContext(ctx, "EXPLAIN FORMAT=JSON "+statement)
+ if err != nil {
+ return 0, err
+ }
+ defer rows.Close()
+ plan, err := readExplainJSON(rows)
+ if err != nil {
+ return 0, err
+ }
+ return mysqlparser.EstimateAffectedRowsFromOceanBaseExplainJSON(plan)
}
- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)
- rows, err := d.db.QueryContext(ctx, explainSQL)
+ list, err := mysqlparser.ParseMySQL(statement)
if err != nil {
- return 0, err
+ return 0, errors.Wrap(err, "failed to parse statement")
}
- defer rows.Close()
-
- // mysql> explain delete from td;
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- // | 1 | DELETE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | NULL |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- //
- // mysql> explain insert into td select * from td;
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
- // | 1 | INSERT | td | NULL | ALL | NULL | NULL | NULL | NULL | NULL | NULL | NULL |
- // | 1 | SIMPLE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | Using temporary |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
- columns, err := rows.Columns()
+ if list == nil || len(list.Items) != 1 {
+ return 0, errors.New("expected exactly one statement")
+ }
+ query := mysqlparser.AffectedRowsQuery(list.Items[0], statement)
+ plan, err := ExplainJSON(ctx, d.db, query)
if err != nil {
return 0, err
}
- rowsIndex, ok := util.GetColumnIndex(columns, "rows")
- if !ok {
- return 0, nil
- }
- for rows.Next() {
- scanArgs := make([]any, len(columns))
- for i := range scanArgs {
- var unused any
- scanArgs[i] = &unused
- }
- var rowsColumn sql.NullInt64
- scanArgs[rowsIndex] = &rowsColumn
- if err := rows.Scan(scanArgs...); err != nil {
- return 0, err
- }
+ return EstimateAffectedRows(ctx, d.db, list.Items[0], query, plan)
+}
- if rowsColumn.Valid {
- return capAffectedRowsByLimit(rowsColumn.Int64, statement), nil
- }
+// EstimateAffectedRows returns the rows stmt modifies according to plan, the ExplainJSON output for
+// query, which is the statement's AffectedRowsQuery. When plan has no estimate to read, as MySQL 8.0
+// prints no source plan for an INSERT ... SELECT that computes window functions, the estimate of the
+// tabular EXPLAIN stands in for it.
+func EstimateAffectedRows(ctx context.Context, db *sql.DB, stmt ast.Node, query string, plan string) (int64, error) {
+ count, err := mysqlparser.EstimateAffectedRowsFromExplainJSON(stmt, plan)
+ if err == nil {
+ return count, nil
}
- if err := rows.Err(); err != nil {
+ rows, tables, ok := explainTabularEstimate(ctx, db, query)
+ if !ok {
return 0, err
}
- return 0, nil
+ // A multi-table UPDATE or DELETE can change a row of each target for every row its join produces.
+ return mysqlparser.CapAffectedRowsByLimit(stmt, rows*float64(mysqlparser.DMLTargetCount(stmt, tables))), nil
}
-func (d *Driver) countAffectedRowsFromJSONPlan(ctx context.Context, statement string) (int64, bool) {
- rows, err := d.db.QueryContext(ctx, fmt.Sprintf("EXPLAIN FORMAT=JSON %s", statement))
+// explainTabularEstimate returns the rows that the first query block of the tabular EXPLAIN of query
+// produces, and how many tables it joins: the product, over those tables, of the rows each reads
+// scaled by its filtered percentage where the server prints one.
+func explainTabularEstimate(ctx context.Context, db *sql.DB, query string) (float64, int, bool) {
+ // MySQL 9 prints a plain EXPLAIN in the TREE format.
+ rows, err := db.QueryContext(ctx, "EXPLAIN FORMAT=TRADITIONAL "+query)
if err != nil {
- return 0, false
+ return 0, 0, false
}
defer rows.Close()
-
- var plan strings.Builder
- for rows.Next() {
- var planColumn sql.NullString
- if err := rows.Scan(&planColumn); err != nil {
- return 0, false
- }
- if planColumn.Valid {
- plan.WriteString(planColumn.String)
- }
- }
- if err := rows.Err(); err != nil {
- return 0, false
- }
- return mysqlparser.GetEstimatedAffectedRowsFromExplainJSON(plan.String())
-}
-
-func capAffectedRowsByLimit(count int64, statement string) int64 {
- limit, ok := affectedRowsLimit(statement)
- if !ok || limit >= count {
- return count
- }
- return limit
-}
-
-func affectedRowsLimit(statement string) (int64, bool) {
- list, err := mysqlparser.ParseMySQL(statement)
- if err != nil || list == nil || len(list.Items) != 1 {
- return 0, false
- }
-
- var limit *ast.Limit
- switch stmt := list.Items[0].(type) {
- case *ast.UpdateStmt:
- limit = stmt.Limit
- case *ast.DeleteStmt:
- limit = stmt.Limit
- case *ast.InsertStmt:
- if stmt.Select != nil {
- limit = stmt.Select.Limit
- }
- default:
- return 0, false
+ columns, err := rows.Columns()
+ if err != nil {
+ return 0, 0, false
}
- if limit == nil || limit.Count == nil {
- return 0, false
+ column := func(name string) int {
+ return slices.IndexFunc(columns, func(column string) bool { return strings.EqualFold(column, name) })
}
- lit, ok := limit.Count.(*ast.IntLit)
- if !ok || lit.Value < 0 {
- return 0, false
+ idIndex, rowsIndex, filteredIndex := column("id"), column("rows"), column("filtered")
+ if idIndex < 0 || rowsIndex < 0 {
+ return 0, 0, false
}
- return lit.Value, true
-}
-
-func countAffectedRowsForOceanBase(ctx context.Context, sqlDB *sql.DB, dml string) (int64, error) {
- explainSQL := fmt.Sprintf("EXPLAIN FORMAT=JSON %s", dml)
- rows, err := sqlDB.QueryContext(ctx, explainSQL)
- if err != nil {
- return 0, err
+ values := make([]sql.NullString, len(columns))
+ dest := make([]any, len(columns))
+ for i := range values {
+ dest[i] = &values[i]
}
- defer rows.Close()
- var planString strings.Builder
+ var blockID string
+ estimate, tables := 1.0, 0
for rows.Next() {
- var planColumn sql.NullString
- if err := rows.Scan(&planColumn); err != nil {
- return 0, err
+ if err := rows.Scan(dest...); err != nil {
+ return 0, 0, false
}
- if !planColumn.Valid {
+ tableRows, err := strconv.ParseFloat(values[rowsIndex].String, 64)
+ if !values[rowsIndex].Valid || err != nil || (tables > 0 && values[idIndex].String != blockID) {
continue
}
- planString.WriteString(planColumn.String)
- }
- if err := rows.Err(); err != nil {
- return 0, err
- }
- if planString.Len() == 0 {
- return 0, nil
+ blockID = values[idIndex].String
+ tables++
+ estimate *= tableRows
+ if filteredIndex >= 0 && values[filteredIndex].Valid {
+ if filtered, err := strconv.ParseFloat(values[filteredIndex].String, 64); err == nil {
+ estimate *= filtered / 100
+ }
+ }
}
- return getAffectedRowsFromOceanBaseQueryPlan(planString.String())
+ return estimate, tables, tables > 0 && rows.Err() == nil
}
-func getAffectedRowsFromOceanBaseQueryPlan(planString string) (int64, error) {
- var planValue map[string]json.RawMessage
- if err := json.Unmarshal([]byte(planString), &planValue); err != nil {
- return 0, errors.Wrapf(err, "failed to parse query plan from string: %+v", planString)
- }
- if len(planValue) == 0 {
- return 0, nil
- }
- queryPlan := oceanBaseQueryPlan{}
- if err := queryPlan.Unmarshal(planValue); err != nil {
- return 0, errors.Wrapf(err, "failed to parse query plan from map: %+v", planValue)
- }
- if queryPlan.Operator != "" {
- return queryPlan.EstRows, nil
+// ExplainJSON returns the `EXPLAIN FORMAT=JSON` output for the statement in the JSON format
+// version that mysqlparser.EstimateAffectedRowsFromExplainJSON reads.
+func ExplainJSON(ctx context.Context, db *sql.DB, statement string) (string, error) {
+ // The session variable must be set on the connection that runs the EXPLAIN.
+ conn, err := db.Conn(ctx)
+ if err != nil {
+ return "", err
}
- count := int64(-1)
- for k, v := range planValue {
- if !strings.HasPrefix(k, "CHILD_") {
- continue
- }
- child := oceanBaseQueryPlan{}
- if err := child.Unmarshal(v); err != nil {
- return 0, errors.Wrapf(err, "failed to parse field '%s', value: %+v", k, v)
- }
- if child.Operator != "" && child.EstRows > count {
- count = child.EstRows
+ defer conn.Close()
+
+ // MySQL 9.5 defaults explain_json_format_version to 2. MySQL before 8.3 and MariaDB lack the
+ // variable and only produce version 1.
+ if _, err := conn.ExecContext(ctx, "SET SESSION explain_json_format_version = 1"); err != nil {
+ var mysqlErr *mysql.MySQLError
+ if !errors.As(err, &mysqlErr) || mysqlErr.Number != erUnknownSystemVariable {
+ return "", errors.Wrap(err, "failed to set explain_json_format_version")
}
+ } else {
+ // The connection returns to the pool, so later statements on it get the server's version back.
+ defer func() {
+ if _, err := conn.ExecContext(ctx, "SET SESSION explain_json_format_version = DEFAULT"); err != nil {
+ slog.Warn("failed to reset explain_json_format_version", log.BBError(err))
+ }
+ }()
}
- if count >= 0 {
- return count, nil
- }
- return 0, nil
-}
-// oceanBaseQueryPlan represents the query plan of OceanBase.
-type oceanBaseQueryPlan struct {
- ID int `json:"ID"`
- Operator string `json:"OPERATOR"`
- Name string `json:"NAME"`
- EstRows int64 `json:"EST.ROWS"`
- Cost int `json:"COST"`
- OutPut any `json:"output"`
+ rows, err := conn.QueryContext(ctx, "EXPLAIN FORMAT=JSON "+statement)
+ if err != nil {
+ return "", err
+ }
+ defer rows.Close()
+ return readExplainJSON(rows)
}
-// Unmarshal parses data and stores the result to current oceanBaseQueryPlan.
-func (plan *oceanBaseQueryPlan) Unmarshal(data any) error {
- b, err := json.Marshal(data)
- if err != nil {
- return err
+// readExplainJSON concatenates the plan rows; OceanBase splits its JSON plan across rows.
+func readExplainJSON(rows *sql.Rows) (string, error) {
+ var plan strings.Builder
+ for rows.Next() {
+ var line sql.NullString
+ if err := rows.Scan(&line); err != nil {
+ return "", err
+ }
+ plan.WriteString(line.String)
}
- if b != nil {
- return json.Unmarshal(b, &plan)
+ if err := rows.Err(); err != nil {
+ return "", err
}
- return nil
+ return plan.String(), nil
}
diff --git a/backend/plugin/db/mysql/stats_test.go b/backend/plugin/db/mysql/stats_test.go
index 0ca99486b6..f13615ba78 100644
--- a/backend/plugin/db/mysql/stats_test.go
+++ b/backend/plugin/db/mysql/stats_test.go
@@ -8,15 +8,32 @@ import (
"strings"
"testing"
+ "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
"github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
var testOceanBaseExplainRows [][]driver.Value
-var testMySQLExplainRows [][]driver.Value
-// testMySQLExplainJSON is returned for `EXPLAIN FORMAT=JSON` queries; when empty, the
-// JSON query returns no rows and CountAffectedRows falls back to the tabular rows.
-var testMySQLExplainJSON string
+var (
+ // testMySQLExplainJSON is the plan the fake MySQL server returns for EXPLAIN FORMAT=JSON.
+ testMySQLExplainJSON string
+ // testMySQLExplainTable is the plan the fake MySQL server returns for a tabular EXPLAIN, with the
+ // columns in testMySQLExplainTableColumns.
+ testMySQLExplainTable [][]driver.Value
+ // testMySQLSetErr is the error the fake MySQL server returns for SET statements.
+ testMySQLSetErr error
+ // testMySQLStatements records every statement the fake MySQL server receives, in order.
+ testMySQLStatements []testMySQLStatement
+ testMySQLConnections int
+)
+
+type testMySQLStatement struct {
+ conn int
+ query string
+}
func init() {
sql.Register("test_oceanbase_explain", testOceanBaseExplainDriver{})
@@ -44,80 +61,76 @@ func (testOceanBaseExplainConn) Begin() (driver.Tx, error) {
}
func (testOceanBaseExplainConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {
- return &testOceanBaseExplainResultRows{rows: testOceanBaseExplainRows}, nil
-}
-
-type testOceanBaseExplainResultRows struct {
- rows [][]driver.Value
- idx int
-}
-
-func (*testOceanBaseExplainResultRows) Columns() []string {
- return []string{"Query Plan"}
-}
-
-func (*testOceanBaseExplainResultRows) Close() error {
- return nil
-}
-
-func (r *testOceanBaseExplainResultRows) Next(dest []driver.Value) error {
- if r.idx >= len(r.rows) {
- return io.EOF
- }
- copy(dest, r.rows[r.idx])
- r.idx++
- return nil
+ return &testExplainResultRows{rows: testOceanBaseExplainRows}, nil
}
type testMySQLExplainDriver struct{}
func (testMySQLExplainDriver) Open(string) (driver.Conn, error) {
- return testMySQLExplainConn{}, nil
+ testMySQLConnections++
+ return &testMySQLExplainConn{id: testMySQLConnections}, nil
}
-type testMySQLExplainConn struct{}
+type testMySQLExplainConn struct {
+ id int
+}
-func (testMySQLExplainConn) Prepare(string) (driver.Stmt, error) {
+func (*testMySQLExplainConn) Prepare(string) (driver.Stmt, error) {
return nil, driver.ErrSkip
}
-func (testMySQLExplainConn) Close() error {
+func (*testMySQLExplainConn) Close() error {
return nil
}
-func (testMySQLExplainConn) Begin() (driver.Tx, error) {
+func (*testMySQLExplainConn) Begin() (driver.Tx, error) {
return nil, driver.ErrSkip
}
-func (testMySQLExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
- if strings.HasPrefix(query, "EXPLAIN FORMAT=JSON") {
- var rows [][]driver.Value
- if testMySQLExplainJSON != "" {
- rows = [][]driver.Value{{testMySQLExplainJSON}}
- }
- return &testMySQLExplainResultRows{columns: []string{"EXPLAIN"}, rows: rows}, nil
+func (c *testMySQLExplainConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {
+ testMySQLStatements = append(testMySQLStatements, testMySQLStatement{conn: c.id, query: query})
+ if !strings.HasPrefix(query, "SET ") {
+ return nil, errors.Errorf("unexpected exec %q", query)
+ }
+ if testMySQLSetErr != nil {
+ return nil, testMySQLSetErr
+ }
+ return driver.ResultNoRows, nil
+}
+
+var testMySQLExplainTableColumns = []string{"id", "select_type", "table", "rows", "filtered"}
+
+func (c *testMySQLExplainConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+ testMySQLStatements = append(testMySQLStatements, testMySQLStatement{conn: c.id, query: query})
+ switch {
+ case strings.HasPrefix(query, "EXPLAIN FORMAT=JSON "):
+ return &testExplainResultRows{rows: [][]driver.Value{{testMySQLExplainJSON}}}, nil
+ case strings.HasPrefix(query, "EXPLAIN "):
+ return &testExplainResultRows{columns: testMySQLExplainTableColumns, rows: testMySQLExplainTable}, nil
+ default:
+ return nil, errors.Errorf("unexpected query %q", query)
}
- return &testMySQLExplainResultRows{
- columns: []string{"id", "select_type", "table", "type", "rows", "filtered"},
- rows: testMySQLExplainRows,
- }, nil
}
-type testMySQLExplainResultRows struct {
+type testExplainResultRows struct {
+ // columns defaults to the single EXPLAIN column of a JSON plan.
columns []string
rows [][]driver.Value
idx int
}
-func (r *testMySQLExplainResultRows) Columns() []string {
- return r.columns
+func (r *testExplainResultRows) Columns() []string {
+ if r.columns != nil {
+ return r.columns
+ }
+ return []string{"EXPLAIN"}
}
-func (*testMySQLExplainResultRows) Close() error {
+func (*testExplainResultRows) Close() error {
return nil
}
-func (r *testMySQLExplainResultRows) Next(dest []driver.Value) error {
+func (r *testExplainResultRows) Next(dest []driver.Value) error {
if r.idx >= len(r.rows) {
return io.EOF
}
@@ -126,74 +139,220 @@ func (r *testMySQLExplainResultRows) Next(dest []driver.Value) error {
return nil
}
-func TestCountAffectedRowsCapsExplainEstimateByLimit(t *testing.T) {
- testMySQLExplainJSON = ""
- testMySQLExplainRows = [][]driver.Value{
- {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},
- }
+// newTestMySQLDriver returns a driver over the fake MySQL server whose pool keeps no idle
+// connections, so statements that do not share a pinned connection land on different ones.
+func newTestMySQLDriver(t *testing.T, plan string) *Driver {
+ t.Helper()
+ testMySQLExplainJSON = plan
+ testMySQLExplainTable = nil
+ testMySQLSetErr = nil
+ testMySQLStatements = nil
db, err := sql.Open("test_mysql_explain", "")
require.NoError(t, err)
- defer db.Close()
+ db.SetMaxIdleConns(0)
+ t.Cleanup(func() { _ = db.Close() })
+ return &Driver{dbType: storepb.Engine_MYSQL, db: db}
+}
- driver := &Driver{db: db}
+const (
+ // The flagged UPDATE target is scanned first; the join filter on s comes after it.
+ testUpdateTargetFirstPlan = `{"query_block":{"select_id":1,"nested_loop":[
+ {"table":{"update":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00"}},
+ {"table":{"table_name":"s","access_type":"eq_ref","rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00"}}
+ ]}}`
+ testInsertSelectPlan = `{"query_block":{"select_id":1,
+ "table":{"insert":true,"table_name":"t2","access_type":"ALL"},
+ "insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}
+ }}`
+)
+func TestCountAffectedRowsEstimatesFromJSONPlan(t *testing.T) {
+ // A single-table plan of the SELECT a single-table UPDATE or DELETE is explained as.
+ const filteredScanPlan = `{"query_block":{"select_id":1,"table":{"table_name":"td","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00"}}}`
for _, tc := range []struct {
+ name string
+ plan string
statement string
+ explained string
want int64
}{
{
- statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 10;",
+ name: "update bounded by the join after the target",
+ plan: testUpdateTargetFirstPlan,
+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",
+ explained: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",
+ want: 1133,
+ },
+ {
+ name: "single-table update explained as a select",
+ plan: filteredScanPlan,
+ statement: "UPDATE td SET c = 1 WHERE d = 0;",
+ explained: "SELECT 1 FROM td WHERE d = 0",
+ want: 100,
+ },
+ {
+ name: "update capped by LIMIT",
+ plan: filteredScanPlan,
+ statement: "UPDATE td SET c = 1 WHERE d = 0 LIMIT 10;",
+ explained: "SELECT 1 FROM td WHERE d = 0 LIMIT 10",
want: 10,
},
{
- statement: "DELETE FROM td WHERE c = 0 LIMIT 20;",
+ name: "delete capped by LIMIT",
+ plan: filteredScanPlan,
+ statement: "DELETE FROM td WHERE d = 0 ORDER BY id LIMIT 20;",
+ explained: "SELECT 1 FROM td WHERE d = 0 ORDER BY id LIMIT 20",
want: 20,
},
{
- statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 30;",
- want: 30,
+ name: "insert select reads the source plan",
+ plan: testInsertSelectPlan,
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ explained: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ want: 100,
},
{
- statement: "UPDATE td SET c = 1 WHERE c = 0;",
- want: 1000,
+ name: "insert select capped by LIMIT",
+ plan: testInsertSelectPlan,
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 30;",
+ explained: "INSERT INTO t2 SELECT * FROM t WHERE d = 3 LIMIT 30;",
+ want: 30,
},
} {
- got, err := driver.CountAffectedRows(context.Background(), tc.statement)
- require.NoError(t, err)
- require.Equal(t, tc.want, got, tc.statement)
+ t.Run(tc.name, func(t *testing.T) {
+ d := newTestMySQLDriver(t, tc.plan)
+ got, err := d.CountAffectedRows(context.Background(), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ require.Equal(t, "EXPLAIN FORMAT=JSON "+tc.explained, testMySQLStatements[1].query)
+ })
}
}
-func TestCountAffectedRowsPrefersJSONPlanTargetNode(t *testing.T) {
- // The tabular EXPLAIN's first row is a driving-table scan estimate; the JSON plan
- // flags the update target whose cumulative estimate is far smaller (BYT-9858).
- testMySQLExplainJSON = `{"query_block":{"select_id":1,"nested_loop":[
- {"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":100232,"rows_produced_per_join":3006,"filtered":"3.00"}},
- {"table":{"update":true,"table_name":"o","access_type":"ref","rows_examined_per_scan":3,"rows_produced_per_join":1144,"filtered":"10.00"}}
- ]}}`
- testMySQLExplainRows = [][]driver.Value{
- {int64(1), "SIMPLE", "t", "ALL", int64(100232), "3.00"},
- {int64(1), "UPDATE", "o", "ref", int64(3), "10.00"},
+func TestCountAffectedRowsScopesJSONFormatVersionToTheExplain(t *testing.T) {
+ const statement = "INSERT INTO t2 SELECT * FROM t WHERE d = 3;"
+ for _, tc := range []struct {
+ name string
+ setErr error
+ wantStatements []string
+ }{
+ {
+ name: "server with explain_json_format_version",
+ wantStatements: []string{
+ "SET SESSION explain_json_format_version = 1",
+ "EXPLAIN FORMAT=JSON " + statement,
+ "SET SESSION explain_json_format_version = DEFAULT",
+ },
+ },
+ {
+ name: "server without explain_json_format_version",
+ setErr: &mysql.MySQLError{Number: 1193, Message: "Unknown system variable 'explain_json_format_version'"},
+ wantStatements: []string{
+ "SET SESSION explain_json_format_version = 1",
+ "EXPLAIN FORMAT=JSON " + statement,
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ d := newTestMySQLDriver(t, testInsertSelectPlan)
+ testMySQLSetErr = tc.setErr
+
+ got, err := d.CountAffectedRows(context.Background(), statement)
+ require.NoError(t, err)
+ require.Equal(t, int64(100), got)
+ var statements []string
+ for _, s := range testMySQLStatements {
+ statements = append(statements, s.query)
+ require.Equal(t, testMySQLStatements[0].conn, s.conn)
+ }
+ require.Equal(t, tc.wantStatements, statements)
+ })
}
- defer func() { testMySQLExplainJSON = "" }()
- db, err := sql.Open("test_mysql_explain", "")
- require.NoError(t, err)
- defer db.Close()
+ t.Run("other SET failures stop the estimate", func(t *testing.T) {
+ d := newTestMySQLDriver(t, testInsertSelectPlan)
+ testMySQLSetErr = &mysql.MySQLError{Number: 1227, Message: "Access denied"}
- driver := &Driver{db: db}
+ _, err := d.CountAffectedRows(context.Background(), statement)
+ require.ErrorContains(t, err, "failed to set explain_json_format_version")
+ require.Len(t, testMySQLStatements, 1)
+ })
+}
- got, err := driver.CountAffectedRows(context.Background(), "UPDATE o SET c = 1 WHERE c IS NULL AND EXISTS (SELECT 1 FROM t WHERE t.k = o.k);")
- require.NoError(t, err)
- require.Equal(t, int64(1144), got)
+func TestCountAffectedRowsFallsBackToTabularPlan(t *testing.T) {
+ // MySQL 8.0 prints no source plan for an INSERT ... SELECT that computes window functions.
+ const plan = `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`
+ for _, tc := range []struct {
+ name string
+ statement string
+ // table is the tabular plan, which defaults to a single table with 1000 rows and 33.33% filtered.
+ table [][]driver.Value
+ want int64
+ }{
+ {
+ name: "rows of the first table estimated, scaled by filtered",
+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER (ORDER BY id) FROM t WHERE c < 10;",
+ want: 333,
+ },
+ {
+ name: "capped by LIMIT",
+ statement: "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER (ORDER BY id) FROM t WHERE c < 10 LIMIT 50;",
+ want: 50,
+ },
+ {
+ // 3 small rows each join 100 big rows; the subquery's own block does not count.
+ name: "rows joined in the first query block",
+ statement: "INSERT INTO t2 SELECT big.id, ROW_NUMBER() OVER () FROM small JOIN big ON big.s_id = small.id WHERE small.id IN (SELECT id FROM s);",
+ table: [][]driver.Value{
+ {int64(1), "INSERT", "t2", nil, nil},
+ {int64(1), "SIMPLE", "small", int64(3), 100.0},
+ {int64(1), "SIMPLE", "big", int64(100), 100.0},
+ {int64(2), "SUBQUERY", "s", int64(100), 10.0},
+ },
+ want: 300,
+ },
+ {
+ // Each of the 300 joined rows changes a row of a and a row of b.
+ name: "multi-table update counts each target",
+ statement: "UPDATE small JOIN big ON big.s_id = small.id SET small.x = 1, big.v = 1;",
+ table: [][]driver.Value{
+ {int64(1), "UPDATE", "small", int64(3), 100.0},
+ {int64(1), "UPDATE", "big", int64(100), 100.0},
+ },
+ want: 600,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ d := newTestMySQLDriver(t, plan)
+ testMySQLExplainTable = tc.table
+ if testMySQLExplainTable == nil {
+ testMySQLExplainTable = [][]driver.Value{
+ {int64(1), "INSERT", "t2", nil, nil},
+ {int64(1), "SIMPLE", "t", int64(1000), 33.33},
+ }
+ }
+ got, err := d.CountAffectedRows(context.Background(), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ require.Equal(t, "EXPLAIN FORMAT=TRADITIONAL "+tc.statement, testMySQLStatements[len(testMySQLStatements)-1].query)
+ })
+ }
+}
- // The statement LIMIT still caps the JSON-plan estimate.
- got, err = driver.CountAffectedRows(context.Background(), "UPDATE o SET c = 1 WHERE c IS NULL LIMIT 10;")
- require.NoError(t, err)
- require.Equal(t, int64(10), got)
+func TestCountAffectedRowsReportsUninterpretablePlan(t *testing.T) {
+ // A plan without an estimate, with no tabular estimate either, fails rather than reading as zero rows.
+ d := newTestMySQLDriver(t, `{"query_block":{"select_id":1,"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}`)
+ _, err := d.CountAffectedRows(context.Background(), "INSERT INTO t2 SELECT id, ROW_NUMBER() OVER () FROM t;")
+ require.ErrorContains(t, err, `table "t2" in the plan has no row estimate`)
}
-func TestCountAffectedRowsForOceanBaseConcatenatesExplainRows(t *testing.T) {
+func TestCountAffectedRowsForOceanBase(t *testing.T) {
+ db, err := sql.Open("test_oceanbase_explain", "")
+ require.NoError(t, err)
+ defer db.Close()
+ d := &Driver{dbType: storepb.Engine_OCEANBASE, db: db}
+
+ // OceanBase splits the JSON plan across rows.
testOceanBaseExplainRows = [][]driver.Value{
{`{`},
{` "ID":0,`},
@@ -212,11 +371,15 @@ func TestCountAffectedRowsForOceanBaseConcatenatesExplainRows(t *testing.T) {
{` }`},
{`}`},
}
- db, err := sql.Open("test_oceanbase_explain", "")
- require.NoError(t, err)
- defer db.Close()
-
- got, err := countAffectedRowsForOceanBase(context.Background(), db, "update dba_test_1 set log_id=1 where id < 1000;")
+ got, err := d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")
require.NoError(t, err)
require.Equal(t, int64(1000), got)
+
+ testOceanBaseExplainRows = [][]driver.Value{{`{"ID":0,"OPERATOR":"UPDATE","NAME":""}`}}
+ _, err = d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")
+ require.ErrorContains(t, err, `operator "UPDATE" has no EST.ROWS`)
+
+ testOceanBaseExplainRows = nil
+ _, err = d.CountAffectedRows(context.Background(), "update dba_test_1 set log_id=1 where id < 1000;")
+ require.Error(t, err)
}
diff --git a/backend/plugin/db/oracle/stats.go b/backend/plugin/db/oracle/stats.go
index d7231b8582..3c1dcaed6d 100644
--- a/backend/plugin/db/oracle/stats.go
+++ b/backend/plugin/db/oracle/stats.go
@@ -2,67 +2,196 @@ package oracle
import (
"context"
+ "crypto/rand"
"database/sql"
"fmt"
- "strconv"
+ "log/slog"
+ "math/big"
+ "slices"
"strings"
+ "time"
"github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/utils"
)
+// dynamicSamplingHint makes the optimizer sample the tables while it plans the statement, so its
+// estimates account for skewed columns without histograms and for correlated predicates. Unlike a
+// DYNAMIC_SAMPLING hint, which covers only its own query block, OPT_PARAM covers every block.
+const dynamicSamplingHint = "OPT_PARAM('optimizer_dynamic_sampling' 11)"
+
+// CountAffectedRows returns the optimizer's estimate of the rows an UPDATE, DELETE, INSERT, or MERGE
+// changes, planned with dynamic sampling.
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
- // Statement includes trailing semicolon, so we need to remove it.
- if _, err := d.db.ExecContext(ctx, fmt.Sprintf("EXPLAIN PLAN FOR %s", statement)); err != nil {
- return 0, err
+ // Older Oracle releases reject a trailing semicolon with ORA-00911.
+ statement = strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon)
+ randNum, err := rand.Int(rand.Reader, big.NewInt(999))
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to generate random statement ID")
}
- rows, err := d.db.QueryContext(ctx, "SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY)")
+ statementID := fmt.Sprintf("%d%d", time.Now().UnixMilli(), randNum.Int64())
+
+ // The default PLAN_TABLE is a session-private temporary table, so explaining,
+ // reading, and deleting the plan must share one connection.
+ conn, err := d.db.Conn(ctx)
if err != nil {
- return 0, err
+ return 0, errors.Wrapf(err, "failed to get connection")
}
- defer rows.Close()
+ defer conn.Close()
- // EXPLAIN PLAN FOR UPDATE TEST SET id = 121 WHERE id = 242; SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY);
- // | Id | Operation | Name | Rows | Bytes | Cost (%CPU)| Time |
- // ---------------------------------------------------------------------------
- // | 0 | UPDATE STATEMENT | | 1 | 13 | 3 (0)| 00:00:01 |
- // | 1 | UPDATE | TEST | | | | |
- // |* 2 | TABLE ACCESS FULL| TEST | 1 | 13 | 3 (0)| 00:00:01 |
- var rowsIndex int
- foundRowsIndex := false
+ if _, err := conn.ExecContext(ctx, fmt.Sprintf("EXPLAIN PLAN SET STATEMENT_ID = '%s' FOR %s", statementID, withDynamicSamplingHint(statement))); err != nil {
+ return 0, errors.Wrapf(err, "failed to explain statement")
+ }
+ defer func() {
+ if _, err := conn.ExecContext(ctx, fmt.Sprintf("DELETE FROM PLAN_TABLE WHERE STATEMENT_ID = '%s'", statementID)); err != nil {
+ slog.Warn("failed to delete explain plan rows", log.BBError(err))
+ }
+ }()
+
+ rows, err := conn.QueryContext(ctx, fmt.Sprintf("SELECT ID, PARENT_ID, OPERATION, CARDINALITY FROM PLAN_TABLE WHERE STATEMENT_ID = '%s' ORDER BY ID", statementID))
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to get plan cardinality")
+ }
+ defer rows.Close()
+ var plan []planRow
for rows.Next() {
- var planRow sql.NullString
- if err := rows.Scan(&planRow); err != nil {
- return 0, err
+ var row planRow
+ if err := rows.Scan(&row.id, &row.parentID, &row.operation, &row.cardinality); err != nil {
+ return 0, errors.Wrapf(err, "failed to get plan cardinality")
}
- if !planRow.Valid {
+ plan = append(plan, row)
+ }
+ if err := rows.Err(); err != nil {
+ return 0, errors.Wrapf(err, "failed to get plan cardinality")
+ }
+ return getAffectedRowsFromPlan(plan, isInsertFirst(statement))
+}
+
+type planRow struct {
+ id int64
+ parentID sql.NullInt64
+ operation string
+ cardinality sql.NullInt64
+}
+
+// getAffectedRowsFromPlan returns the cardinality of the statement row (ID 0). For a MERGE it returns
+// the cardinality of the rows the MERGE operation reads instead, because with dynamic sampling the
+// statement row of a MERGE plan no longer estimates them. A multi-table INSERT ALL can write each row
+// it reads to every INTO operation, so its estimate is the statement row times those operations;
+// insertFirst marks an INSERT FIRST, which writes each row at most once.
+func getAffectedRowsFromPlan(plan []planRow, insertFirst bool) (int64, error) {
+ for _, row := range plan {
+ if row.operation != "MERGE" {
continue
}
- tokens := strings.Split(planRow.String, "|")
- if !foundRowsIndex {
- for i, token := range tokens {
- token = strings.Trim(token, " ")
- if token == "Rows" {
- rowsIndex, foundRowsIndex = i, true
- break
- }
+ // The first child chain below MERGE passes through views that have no cardinality.
+ for parentID := row.id; ; {
+ i := slices.IndexFunc(plan, func(r planRow) bool { return r.parentID.Valid && r.parentID.Int64 == parentID })
+ if i < 0 || plan[i].id <= parentID {
+ return 0, errors.New("the MERGE operation in the plan has no cardinality estimate")
}
- continue
- }
- if rowsIndex >= len(tokens) {
- continue
- }
- rowsToken := strings.Trim(tokens[rowsIndex], " ")
- if rowsToken == "" {
- continue
+ if plan[i].cardinality.Valid {
+ return plan[i].cardinality.Int64, nil
+ }
+ parentID = plan[i].id
}
- v, err := strconv.ParseInt(rowsToken, 10, 64)
- if err != nil {
- return 0, errors.Errorf("failed to get integer from %q", rowsToken)
+ }
+ i := slices.IndexFunc(plan, func(r planRow) bool { return r.id == 0 })
+ if i < 0 {
+ return 0, errors.New("plan has no statement row")
+ }
+ if !plan[i].cardinality.Valid {
+ return 0, errors.New("plan has no cardinality estimate")
+ }
+ rows := plan[i].cardinality.Int64
+ targets := 0
+ for _, row := range plan {
+ if row.operation == "INTO" {
+ targets++
}
- return v, nil
}
- if err := rows.Err(); err != nil {
- return 0, err
+ if targets > 1 && !insertFirst {
+ return common.RoundRows(float64(rows) * float64(targets)), nil
+ }
+ return rows, nil
+}
+
+// isInsertFirst reports whether statement is an INSERT FIRST.
+func isInsertFirst(statement string) bool {
+ start := skipSpaceAndComments(statement)
+ end := start
+ for end < len(statement) && isLetter(statement[end]) {
+ end++
+ }
+ if !strings.EqualFold(statement[start:end], "INSERT") {
+ return false
+ }
+ start = end + skipSpaceAndComments(statement[end:])
+ end = start
+ for end < len(statement) && isLetter(statement[end]) {
+ end++
+ }
+ return strings.EqualFold(statement[start:end], "FIRST")
+}
+
+// withDynamicSamplingHint adds dynamicSamplingHint to the hint of an UPDATE, DELETE, INSERT, or MERGE
+// statement and returns other statements unchanged.
+func withDynamicSamplingHint(statement string) string {
+ start := skipSpaceAndComments(statement)
+ end := start
+ for end < len(statement) && isLetter(statement[end]) {
+ end++
+ }
+ switch strings.ToUpper(statement[start:end]) {
+ case "UPDATE", "DELETE", "INSERT", "MERGE":
+ default:
+ return statement
}
- return 0, nil
+ // Oracle reads only the first hint after the keyword, so the hint joins an existing one.
+ hint := end
+ for hint < len(statement) && isSpace(statement[hint]) {
+ hint++
+ }
+ if strings.HasPrefix(statement[hint:], "/*+") || strings.HasPrefix(statement[hint:], "--+") {
+ return statement[:hint+3] + " " + dynamicSamplingHint + " " + statement[hint+3:]
+ }
+ return statement[:end] + " /*+ " + dynamicSamplingHint + " */" + statement[end:]
+}
+
+// skipSpaceAndComments returns the offset of the first byte of the statement that is not whitespace
+// or part of a comment.
+func skipSpaceAndComments(statement string) int {
+ i := 0
+ for i < len(statement) {
+ switch {
+ case isSpace(statement[i]):
+ i++
+ case strings.HasPrefix(statement[i:], "--"):
+ end := strings.IndexByte(statement[i:], '\n')
+ if end < 0 {
+ return len(statement)
+ }
+ i += end + 1
+ case strings.HasPrefix(statement[i:], "/*"):
+ end := strings.Index(statement[i+2:], "*/")
+ if end < 0 {
+ return len(statement)
+ }
+ i += end + 4
+ default:
+ return i
+ }
+ }
+ return i
+}
+
+func isSpace(c byte) bool {
+ return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
+}
+
+func isLetter(c byte) bool {
+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
}
diff --git a/backend/plugin/db/oracle/stats_test.go b/backend/plugin/db/oracle/stats_test.go
new file mode 100644
index 0000000000..70b066350d
--- /dev/null
+++ b/backend/plugin/db/oracle/stats_test.go
@@ -0,0 +1,247 @@
+package oracle
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "io"
+ "regexp"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestCountAffectedRows(t *testing.T) {
+ const hint = "/*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */"
+ for _, tc := range []struct {
+ name string
+ statement string
+ // plan holds the ID, PARENT_ID, OPERATION, and CARDINALITY of each PLAN_TABLE row.
+ plan [][]driver.Value
+ explained string
+ want int64
+ wantErr bool
+ }{
+ {
+ name: "large_cardinality",
+ statement: "UPDATE huge SET c = 1 WHERE c < 10",
+ // go-ora returns NUMBER columns as strings.
+ plan: [][]driver.Value{
+ {"0", nil, "UPDATE STATEMENT", "5000000"},
+ {"1", "0", "UPDATE", nil},
+ {"2", "1", "TABLE ACCESS", "5000000"},
+ },
+ explained: "UPDATE " + hint + " huge SET c = 1 WHERE c < 10",
+ want: 5000000,
+ },
+ {
+ name: "trailing_semicolon",
+ statement: "DELETE FROM t WHERE c < 10;\n",
+ plan: [][]driver.Value{{"0", nil, "DELETE STATEMENT", "101"}},
+ explained: "DELETE " + hint + " FROM t WHERE c < 10",
+ want: 101,
+ },
+ {
+ // Captured from Oracle 23 with dynamic sampling, where the statement row estimates 1 row
+ // for a MERGE that inserts 100.
+ name: "merge_reads_the_rows_below_the_merge_operation",
+ statement: "MERGE INTO t2 USING (SELECT * FROM t WHERE c < 10) src ON (t2.id = src.id) WHEN NOT MATCHED THEN INSERT VALUES (src.id, src.c, src.d, src.grp, src.v)",
+ plan: [][]driver.Value{
+ {"0", nil, "MERGE STATEMENT", "1"},
+ {"1", "0", "MERGE", nil},
+ {"2", "1", "VIEW", nil},
+ {"3", "2", "NESTED LOOPS", "100"},
+ {"4", "3", "TABLE ACCESS", "100"},
+ {"5", "3", "TABLE ACCESS", "1"},
+ {"6", "5", "INDEX", "1"},
+ },
+ explained: "MERGE " + hint + " INTO t2 USING (SELECT * FROM t WHERE c < 10) src ON (t2.id = src.id) WHEN NOT MATCHED THEN INSERT VALUES (src.id, src.c, src.d, src.grp, src.v)",
+ want: 100,
+ },
+ {
+ // Captured from Oracle 23, where the statement row estimates the 1000 rows an INSERT ALL reads
+ // and writes to three tables.
+ name: "insert_all_writes_each_row_to_every_into",
+ statement: "INSERT ALL INTO t1 VALUES (id) INTO t2 VALUES (id) INTO t3 VALUES (id) SELECT id FROM src",
+ plan: [][]driver.Value{
+ {"0", nil, "INSERT STATEMENT", "1000"},
+ {"1", "0", "MULTI-TABLE INSERT", nil},
+ {"2", "1", "TABLE ACCESS", "1000"},
+ {"3", "1", "INTO", nil},
+ {"4", "1", "INTO", nil},
+ {"5", "1", "INTO", nil},
+ },
+ explained: "INSERT " + hint + " ALL INTO t1 VALUES (id) INTO t2 VALUES (id) INTO t3 VALUES (id) SELECT id FROM src",
+ want: 3000,
+ },
+ {
+ name: "insert_first_writes_each_row_once",
+ statement: "INSERT /* move */ FIRST WHEN id < 10 THEN INTO t1 VALUES (id) ELSE INTO t2 VALUES (id) SELECT id FROM src",
+ plan: [][]driver.Value{
+ {"0", nil, "INSERT STATEMENT", "1000"},
+ {"1", "0", "MULTI-TABLE INSERT", nil},
+ {"2", "1", "TABLE ACCESS", "1000"},
+ {"3", "1", "INTO", nil},
+ {"4", "1", "INTO", nil},
+ },
+ explained: "INSERT " + hint + " /* move */ FIRST WHEN id < 10 THEN INTO t1 VALUES (id) ELSE INTO t2 VALUES (id) SELECT id FROM src",
+ want: 1000,
+ },
+ {
+ name: "merge_without_cardinality",
+ statement: "MERGE INTO t2 USING t ON (t2.id = t.id) WHEN MATCHED THEN UPDATE SET t2.v = t.v",
+ plan: [][]driver.Value{
+ {"0", nil, "MERGE STATEMENT", "1000"},
+ {"1", "0", "MERGE", nil},
+ {"2", "1", "VIEW", nil},
+ },
+ explained: "MERGE " + hint + " INTO t2 USING t ON (t2.id = t.id) WHEN MATCHED THEN UPDATE SET t2.v = t.v",
+ wantErr: true,
+ },
+ {
+ name: "null_cardinality",
+ statement: "DELETE FROM t;",
+ plan: [][]driver.Value{{"0", nil, "DELETE STATEMENT", nil}},
+ explained: "DELETE " + hint + " FROM t",
+ wantErr: true,
+ },
+ {
+ name: "missing_statement_row",
+ statement: "DELETE FROM t;",
+ explained: "DELETE " + hint + " FROM t",
+ wantErr: true,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := &fakePlanConnector{plan: tc.plan}
+ db := sql.OpenDB(connector)
+ defer db.Close()
+ // Without idle connections, every statement that is not pinned to one connection opens a new one.
+ db.SetMaxIdleConns(0)
+
+ got, err := (&Driver{db: db}).CountAffectedRows(context.Background(), tc.statement)
+ if tc.wantErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ }
+
+ require.NotEmpty(t, connector.queries)
+ match := regexp.MustCompile(`^EXPLAIN PLAN SET STATEMENT_ID = '(\d+)' FOR `).FindStringSubmatch(connector.queries[0].text)
+ require.NotNil(t, match, connector.queries[0].text)
+ id := match[1]
+ require.Equal(t, []fakePlanQuery{
+ {conn: 1, text: "EXPLAIN PLAN SET STATEMENT_ID = '" + id + "' FOR " + tc.explained},
+ {conn: 1, text: "SELECT ID, PARENT_ID, OPERATION, CARDINALITY FROM PLAN_TABLE WHERE STATEMENT_ID = '" + id + "' ORDER BY ID"},
+ {conn: 1, text: "DELETE FROM PLAN_TABLE WHERE STATEMENT_ID = '" + id + "'"},
+ }, connector.queries)
+ })
+ }
+}
+
+func TestWithDynamicSamplingHint(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ want string
+ }{
+ {
+ statement: "update t set c = 1",
+ want: "update /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ t set c = 1",
+ },
+ {
+ statement: "INSERT INTO t2 SELECT * FROM t",
+ want: "INSERT /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ INTO t2 SELECT * FROM t",
+ },
+ {
+ statement: "-- clean up\n/* ticket 42 */ DELETE FROM t",
+ want: "-- clean up\n/* ticket 42 */ DELETE /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) */ FROM t",
+ },
+ {
+ statement: "UPDATE /*+ INDEX(t t_c) */ t SET v = 1 WHERE c = 1",
+ want: "UPDATE /*+ OPT_PARAM('optimizer_dynamic_sampling' 11) INDEX(t t_c) */ t SET v = 1 WHERE c = 1",
+ },
+ {
+ statement: "DELETE --+ FULL(t)\nFROM t",
+ want: "DELETE --+ OPT_PARAM('optimizer_dynamic_sampling' 11) FULL(t)\nFROM t",
+ },
+ {
+ statement: "SELECT * FROM t",
+ want: "SELECT * FROM t",
+ },
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ require.Equal(t, tc.want, withDynamicSamplingHint(tc.statement))
+ })
+ }
+}
+
+// fakePlanConnector returns the plan rows for every query and records each statement with the
+// number of the connection that ran it.
+type fakePlanConnector struct {
+ plan [][]driver.Value
+ opened int
+ queries []fakePlanQuery
+}
+
+type fakePlanQuery struct {
+ conn int
+ text string
+}
+
+func (c *fakePlanConnector) Connect(context.Context) (driver.Conn, error) {
+ c.opened++
+ return &fakePlanConn{connector: c, id: c.opened}, nil
+}
+
+func (*fakePlanConnector) Driver() driver.Driver {
+ return nil
+}
+
+type fakePlanConn struct {
+ connector *fakePlanConnector
+ id int
+}
+
+func (*fakePlanConn) Prepare(string) (driver.Stmt, error) {
+ return nil, driver.ErrSkip
+}
+
+func (*fakePlanConn) Close() error {
+ return nil
+}
+
+func (*fakePlanConn) Begin() (driver.Tx, error) {
+ return nil, driver.ErrSkip
+}
+
+func (c *fakePlanConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {
+ c.connector.queries = append(c.connector.queries, fakePlanQuery{conn: c.id, text: query})
+ return driver.RowsAffected(0), nil
+}
+
+func (c *fakePlanConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
+ c.connector.queries = append(c.connector.queries, fakePlanQuery{conn: c.id, text: query})
+ return &fakePlanRows{rows: c.connector.plan}, nil
+}
+
+type fakePlanRows struct {
+ rows [][]driver.Value
+}
+
+func (*fakePlanRows) Columns() []string {
+ return []string{"ID", "PARENT_ID", "OPERATION", "CARDINALITY"}
+}
+
+func (*fakePlanRows) Close() error {
+ return nil
+}
+
+func (r *fakePlanRows) Next(dest []driver.Value) error {
+ if len(r.rows) == 0 {
+ return io.EOF
+ }
+ copy(dest, r.rows[0])
+ r.rows = r.rows[1:]
+ return nil
+}
diff --git a/backend/plugin/db/pg/stats.go b/backend/plugin/db/pg/stats.go
index cdc1ca4da6..9606ab59c1 100644
--- a/backend/plugin/db/pg/stats.go
+++ b/backend/plugin/db/pg/stats.go
@@ -2,59 +2,31 @@ package pg
import (
"context"
- "database/sql"
- "fmt"
- "regexp"
- "strconv"
- "github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
-var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")
-
+// CountAffectedRows returns the planner's estimate of the rows the statement modifies. A search
+// path that base.WithSearchPath put before the statement is set for the EXPLAIN in a transaction
+// that is rolled back.
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)
- rows, err := d.db.QueryContext(ctx, explainSQL)
- if err != nil {
- return 0, err
- }
- defer rows.Close()
-
- // test-bb=# EXPLAIN INSERT INTO t SELECT * FROM t;
- // QUERY PLAN
- // -------------------------------------------------------------
- // Insert on t (cost=0.00..1.03 rows=0 width=0)
- // -> Seq Scan on t t_1 (cost=0.00..1.03 rows=3 width=520)
- // (2 rows)
- //
- // d1=# explain select * from h1;
- // QUERY PLAN
- // ------------------------------------------------------
- // Seq Scan on h1 (cost=0.00..35.50 rows=2550 width=4)
- // (1 row)
- var rowCount int64
- for rows.Next() {
- var planRow sql.NullString
- if err := rows.Scan(&planRow); err != nil {
+ setup, statement := base.SplitSearchPath(statement)
+ queryRow := d.db.QueryRowContext
+ if setup != "" {
+ tx, err := d.db.BeginTx(ctx, nil)
+ if err != nil {
return 0, err
}
-
- if !planRow.Valid {
- continue
- }
-
- matches := rowsRegexp.FindStringSubmatch(planRow.String)
- if len(matches) != 2 {
- continue
- }
- v, err := strconv.ParseInt(matches[1], 10, 64)
- if err != nil {
- return 0, errors.Errorf("failed to get integer from %q", matches[1])
+ defer tx.Rollback()
+ if _, err := tx.ExecContext(ctx, setup); err != nil {
+ return 0, err
}
- rowCount = v
+ queryRow = tx.QueryRowContext
}
- if err := rows.Err(); err != nil {
+ var plan string
+ if err := queryRow(ctx, "EXPLAIN (FORMAT JSON) "+statement).Scan(&plan); err != nil {
return 0, err
}
- return rowCount, nil
+ return pgparser.GetEstimatedAffectedRowsFromExplainJSON(plan)
}
diff --git a/backend/plugin/db/pg/stats_test.go b/backend/plugin/db/pg/stats_test.go
new file mode 100644
index 0000000000..8a356666f5
--- /dev/null
+++ b/backend/plugin/db/pg/stats_test.go
@@ -0,0 +1,76 @@
+package pg
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestCountAffectedRows(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+
+ pgContainer := testcontainer.SharedPgContainer(t)
+ dbName, rawDB := testcontainer.NewPgDatabase(t)
+ _, err := rawDB.ExecContext(ctx, `
+ CREATE TABLE s (id int PRIMARY KEY, flag bool NOT NULL);
+ INSERT INTO s SELECT g, g <= 10 FROM generate_series(1, 100) g;
+ CREATE TABLE big (id int PRIMARY KEY, s_id int NOT NULL, v int);
+ INSERT INTO big SELECT g, (g % 100) + 1, 0 FROM generate_series(1, 10000) g;
+ CREATE TABLE big_archive (LIKE big);
+ ANALYZE;
+ `)
+ require.NoError(t, err)
+
+ openedDriver, err := (&Driver{}).Open(ctx, storepb.Engine_POSTGRES, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Host: pgContainer.GetHost(),
+ Port: pgContainer.GetPort(),
+ Username: "postgres",
+ },
+ Password: "root-password",
+ ConnectionContext: db.ConnectionContext{DatabaseName: dbName},
+ })
+ require.NoError(t, err)
+ defer openedDriver.Close(ctx)
+ driver, ok := openedDriver.(*Driver)
+ require.True(t, ok)
+
+ // Each statement modifies the 1000 big rows joined to the 10 flagged s rows.
+ for _, statement := range []string{
+ "DELETE FROM big USING s WHERE big.s_id = s.id AND s.flag;",
+ "UPDATE big SET v = 1 FROM s WHERE big.s_id = s.id AND s.flag;",
+ "DELETE FROM big WHERE s_id IN (SELECT id FROM s WHERE flag);",
+ "INSERT INTO big_archive SELECT big.* FROM big JOIN s ON big.s_id = s.id WHERE s.flag;",
+ "MERGE INTO big USING s ON big.s_id = s.id AND s.flag WHEN MATCHED THEN UPDATE SET v = 1;",
+ } {
+ rows, err := driver.CountAffectedRows(ctx, statement)
+ require.NoError(t, err, statement)
+ require.Greater(t, rows, int64(500), statement)
+ require.LessOrEqual(t, rows, int64(2000), statement)
+ }
+
+ _, err = driver.CountAffectedRows(ctx, "SELECT * FROM big;")
+ require.Error(t, err)
+
+ _, err = rawDB.ExecContext(ctx, `
+ CREATE SCHEMA app;
+ CREATE TABLE app.only_in_app (id int);
+ INSERT INTO app.only_in_app SELECT generate_series(1, 100);
+ ANALYZE app.only_in_app;
+ `)
+ require.NoError(t, err)
+ // With one connection, the second estimate shows that the search path ended with the first.
+ driver.db.SetMaxOpenConns(1)
+ rows, err := driver.CountAffectedRows(ctx, base.WithSearchPath("DELETE FROM only_in_app;", []string{"app"}))
+ require.NoError(t, err)
+ require.Equal(t, int64(100), rows)
+ _, err = driver.CountAffectedRows(ctx, "DELETE FROM only_in_app;")
+ require.ErrorContains(t, err, "does not exist")
+}
diff --git a/backend/plugin/db/redshift/stats.go b/backend/plugin/db/redshift/stats.go
index 8fbd4e7134..920fbb25ff 100644
--- a/backend/plugin/db/redshift/stats.go
+++ b/backend/plugin/db/redshift/stats.go
@@ -3,55 +3,69 @@ package redshift
import (
"context"
"database/sql"
- "fmt"
"regexp"
"strconv"
"github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
var rowsRegexp = regexp.MustCompile("rows=([0-9]+)")
+// CountAffectedRows returns the planner's estimate of the rows the statement modifies. A search
+// path that base.WithSearchPath put before the statement is set for the EXPLAIN in a transaction
+// that is rolled back.
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)
- rows, err := d.db.QueryContext(ctx, explainSQL)
+ setup, statement := base.SplitSearchPath(statement)
+ query := d.db.QueryContext
+ if setup != "" {
+ tx, err := d.db.BeginTx(ctx, nil)
+ if err != nil {
+ return 0, err
+ }
+ defer tx.Rollback()
+ if _, err := tx.ExecContext(ctx, setup); err != nil {
+ return 0, err
+ }
+ query = tx.QueryContext
+ }
+ rows, err := query(ctx, "EXPLAIN "+statement)
if err != nil {
return 0, err
}
defer rows.Close()
- // # EXPLAIN INSERT INTO t SELECT * FROM t;
- // QUERY PLAN
- // -------------------------------------------------------------
- // XN Seq Scan on t (cost=0.00..0.11 rows=11 width=49)
- //
- // # EXPLAIN SELECT * FROM t;
- // QUERY PLAN
- // ------------------------------------------------------
- // XN Seq Scan on t (cost=0.00..0.11 rows=11 width=49)
- var rowCount int64
+ var plan []string
for rows.Next() {
- var planRow sql.NullString
- if err := rows.Scan(&planRow); err != nil {
+ var line sql.NullString
+ if err := rows.Scan(&line); err != nil {
return 0, err
}
-
- if !planRow.Valid {
- continue
+ if line.Valid {
+ plan = append(plan, line.String)
}
+ }
+ if err := rows.Err(); err != nil {
+ return 0, err
+ }
+ return getAffectedRowsFromPlan(plan)
+}
- matches := rowsRegexp.FindStringSubmatch(planRow.String)
+// getAffectedRowsFromPlan returns the rows estimate of the top plan node. Redshift
+// plans have no DML node, so an INSERT, UPDATE, or DELETE shows only the scans and
+// joins that produce its rows, and the nodes below the top estimate their inputs.
+func getAffectedRowsFromPlan(plan []string) (int64, error) {
+ for _, line := range plan {
+ matches := rowsRegexp.FindStringSubmatch(line)
if len(matches) != 2 {
continue
}
- v, err := strconv.ParseInt(matches[1], 10, 64)
+ rows, err := strconv.ParseInt(matches[1], 10, 64)
if err != nil {
- return 0, errors.Errorf("failed to get integer from %q", matches[1])
+ return 0, errors.Wrapf(err, "failed to parse rows estimate in %q", line)
}
- rowCount = v
- }
- if err := rows.Err(); err != nil {
- return 0, err
+ return rows, nil
}
- return rowCount, nil
+ return 0, errors.Errorf("EXPLAIN plan has no rows estimate: %q", plan)
}
diff --git a/backend/plugin/db/redshift/stats_test.go b/backend/plugin/db/redshift/stats_test.go
new file mode 100644
index 0000000000..086cc7787b
--- /dev/null
+++ b/backend/plugin/db/redshift/stats_test.go
@@ -0,0 +1,77 @@
+package redshift
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// The plans are EXPLAIN output published in the Amazon Redshift docs:
+// https://docs.aws.amazon.com/redshift/latest/dg/c-the-query-plan.html and
+// https://docs.aws.amazon.com/redshift/latest/dg/r_EXPLAIN.html.
+func TestGetAffectedRowsFromPlan(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ plan string
+ want int64
+ wantErr bool
+ }{
+ {
+ name: "hash_join_estimates_joined_rows_not_its_inner_table",
+ plan: ` XN Hash Join DS_BCAST_INNER (cost=0.14..6600286.07 rows=8798 width=84)
+ Hash Cond: ("outer".catid = "inner".catid)
+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=35)
+ -> XN Hash (cost=0.11..0.11 rows=11 width=49)
+ -> XN Seq Scan on category (cost=0.00..0.11 rows=11 width=49)`,
+ want: 8798,
+ },
+ {
+ name: "three_way_join_estimates_the_outermost_join",
+ plan: `XN Hash Join DS_BCAST_INNER (cost=109.98..3871130276.17 rows=172456 width=132)
+ Hash Cond: ("outer".eventid = "inner".eventid)
+ -> XN Merge Join DS_DIST_NONE (cost=0.00..6285.93 rows=172456 width=97)
+ Merge Cond: ("outer".listid = "inner".listid)
+ -> XN Seq Scan on listing (cost=0.00..1924.97 rows=192497 width=44)
+ -> XN Seq Scan on sales (cost=0.00..1724.56 rows=172456 width=53)
+ -> XN Hash (cost=87.98..87.98 rows=8798 width=35)
+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=35)`,
+ want: 172456,
+ },
+ {
+ name: "aggregate_estimates_groups_not_scanned_rows",
+ plan: `XN HashAggregate (cost=131.97..133.41 rows=576 width=17)
+ -> XN Seq Scan on event (cost=0.00..87.98 rows=8798 width=17)`,
+ want: 576,
+ },
+ {
+ name: "scan_estimates_rows_after_its_filter",
+ plan: `XN Seq Scan on venue (cost=0.00..2.02 rows=187 width=45)
+Filter: (venueseats IS NOT NULL)`,
+ want: 187,
+ },
+ {
+ name: "plan_without_rows_estimate",
+ plan: `Filter: (venueseats IS NOT NULL)`,
+ wantErr: true,
+ },
+ {
+ name: "empty_plan",
+ wantErr: true,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var plan []string
+ if tc.plan != "" {
+ plan = strings.Split(tc.plan, "\n")
+ }
+ got, err := getAffectedRowsFromPlan(plan)
+ if tc.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
diff --git a/backend/plugin/db/tidb/stats.go b/backend/plugin/db/tidb/stats.go
index 0572ee9250..6150732fb4 100644
--- a/backend/plugin/db/tidb/stats.go
+++ b/backend/plugin/db/tidb/stats.go
@@ -2,60 +2,144 @@ package tidb
import (
"context"
- "database/sql"
"fmt"
+ "slices"
+ "strconv"
+ "strings"
+ tidbast "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/plugin/db/util"
+ tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb"
)
func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {
- explainSQL := fmt.Sprintf("EXPLAIN %s", statement)
- rows, err := d.db.QueryContext(ctx, explainSQL)
+ rows, err := d.db.QueryContext(ctx, fmt.Sprintf("EXPLAIN %s", statement))
if err != nil {
return 0, err
}
defer rows.Close()
- // mysql> explain delete from td;
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- // | 1 | DELETE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | NULL |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-------+
- //
- // mysql> explain insert into td select * from td;
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
- // | id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
- // | 1 | INSERT | td | NULL | ALL | NULL | NULL | NULL | NULL | NULL | NULL | NULL |
- // | 1 | SIMPLE | td | NULL | ALL | NULL | NULL | NULL | NULL | 1 | 100.00 | Using temporary |
- // +----+-------------+-------+------------+------+---------------+------+---------+------+------+----------+-----------------+
columns, err := rows.Columns()
if err != nil {
return 0, err
}
- rowsIndex, ok := util.GetColumnIndex(columns, "rows")
+ idIndex, ok := util.GetColumnIndex(columns, "id")
if !ok {
- return 0, nil
+ return 0, errors.Errorf("EXPLAIN returned no id column: %v", columns)
+ }
+ estRowsIndex, ok := util.GetColumnIndex(columns, "estRows")
+ if !ok {
+ return 0, errors.Errorf("EXPLAIN returned no estRows column: %v", columns)
}
+ var plan []planRow
for rows.Next() {
scanArgs := make([]any, len(columns))
for i := range scanArgs {
var unused any
scanArgs[i] = &unused
}
- var rowsColumn sql.NullInt64
- scanArgs[rowsIndex] = &rowsColumn
+ var row planRow
+ scanArgs[idIndex] = &row.id
+ scanArgs[estRowsIndex] = &row.estRows
if err := rows.Scan(scanArgs...); err != nil {
return 0, err
}
-
- if rowsColumn.Valid {
- return rowsColumn.Int64, nil
- }
+ plan = append(plan, row)
}
if err := rows.Err(); err != nil {
return 0, err
}
- return 0, nil
+ return getAffectedRows(plan, statement)
+}
+
+// getAffectedRows returns the rows statement modifies according to its plan. The plan estimates the
+// rows the statement reads, and a multi-table UPDATE or DELETE can change a row of each target table
+// for every one of them.
+func getAffectedRows(plan []planRow, statement string) (int64, error) {
+ estimate, err := getAffectedRowsFromPlan(plan)
+ if err != nil {
+ return 0, err
+ }
+ return common.RoundRows(estimate * float64(countDMLTargets(statement))), nil
+}
+
+// countDMLTargets returns how many tables an UPDATE or DELETE can change for each row it reads: the
+// targets a multi-table DELETE lists, or the tables its assignments qualify, counting an unqualified
+// assignment as another joined table, up to the number of joined tables. Other statements change one.
+func countDMLTargets(statement string) int {
+ nodes, err := tidbparser.ParseTiDB(statement, "", "")
+ if err != nil || len(nodes) != 1 {
+ return 1
+ }
+ switch n := nodes[0].(type) {
+ case *tidbast.DeleteStmt:
+ if n.IsMultiTable && n.Tables != nil {
+ return max(len(n.Tables.Tables), 1)
+ }
+ case *tidbast.UpdateStmt:
+ if n.TableRefs == nil {
+ return 1
+ }
+ var tables []string
+ unqualified := 0
+ for _, assignment := range n.List {
+ switch {
+ case assignment.Column == nil || assignment.Column.Table.L == "":
+ unqualified++
+ case !slices.Contains(tables, assignment.Column.Table.L):
+ tables = append(tables, assignment.Column.Table.L)
+ default:
+ }
+ }
+ return max(min(len(tables)+unqualified, countTableSources(n.TableRefs.TableRefs)), 1)
+ default:
+ }
+ return 1
+}
+
+func countTableSources(node tidbast.ResultSetNode) int {
+ switch n := node.(type) {
+ case *tidbast.Join:
+ count := countTableSources(n.Left)
+ if n.Right != nil {
+ count += countTableSources(n.Right)
+ }
+ return count
+ case *tidbast.TableSource:
+ return 1
+ default:
+ return 0
+ }
+}
+
+type planRow struct {
+ id string
+ estRows string
+}
+
+// getAffectedRowsFromPlan returns the estRows of the operator that feeds rows to the
+// INSERT, UPDATE, or DELETE at the plan root. The root itself reports N/A, operators
+// deeper in the tree estimate rows before the filters, joins, and limits above them
+// apply, and a later root plans a CTE or subquery.
+func getAffectedRowsFromPlan(plan []planRow) (float64, error) {
+ if len(plan) == 0 {
+ return 0, errors.New("EXPLAIN returned no plan")
+ }
+ if operator, _, _ := strings.Cut(plan[0].id, "_"); operator != "Insert" && operator != "Update" && operator != "Delete" {
+ return 0, errors.Errorf("EXPLAIN plan root %q is not an INSERT, UPDATE, or DELETE", plan[0].id)
+ }
+ // EXPLAIN lists operators in pre-order and draws each child of the root with a
+ // leading "├─" or "└─", so the next row is the root's first child unless the root
+ // has none, as for INSERT ... SET.
+ if len(plan) == 1 || (!strings.HasPrefix(plan[1].id, "├─") && !strings.HasPrefix(plan[1].id, "└─")) {
+ return 0, nil
+ }
+ estRows, err := strconv.ParseFloat(plan[1].estRows, 64)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to parse estRows of %q", plan[1].id)
+ }
+ return estRows, nil
}
diff --git a/backend/plugin/db/tidb/stats_test.go b/backend/plugin/db/tidb/stats_test.go
new file mode 100644
index 0000000000..8a55e18cc6
--- /dev/null
+++ b/backend/plugin/db/tidb/stats_test.go
@@ -0,0 +1,177 @@
+package tidb
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+)
+
+func TestCountAffectedRows(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ container, database := testcontainer.NewTiDBDatabase(t)
+
+ driver := openTestDriver(ctx, t, container)
+ t.Cleanup(func() {
+ require.NoError(t, driver.Close(ctx))
+ })
+
+ // By default ANALYZE collects only columns that earlier queries filtered on,
+ // which would leave c without statistics.
+ _, err := container.GetDB().ExecContext(ctx, fmt.Sprintf(`
+ CREATE TABLE %[1]s.t (id INT PRIMARY KEY, c INT);
+ CREATE TABLE %[1]s.archive (id INT PRIMARY KEY, c INT);
+ INSERT INTO %[1]s.t
+ WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 1000)
+ SELECT n, n %% 10 FROM seq;
+ ANALYZE TABLE %[1]s.t ALL COLUMNS;
+ `, database))
+ require.NoError(t, err)
+
+ for _, tc := range []struct {
+ name string
+ statement string
+ }{
+ {name: "update", statement: fmt.Sprintf("UPDATE %s.t SET c = 0 WHERE c = 1", database)},
+ {name: "delete", statement: fmt.Sprintf("DELETE FROM %s.t WHERE c = 1", database)},
+ {name: "insert_select", statement: fmt.Sprintf("INSERT INTO %[1]s.archive SELECT * FROM %[1]s.t WHERE c = 1", database)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ got, err := driver.CountAffectedRows(ctx, tc.statement)
+ require.NoError(t, err)
+ // 100 of the 1000 rows match c = 1, while the table scan beneath the
+ // filter estimates all 1000.
+ require.Positive(t, got)
+ require.Less(t, got, int64(1000))
+ })
+ }
+}
+
+// The plans are EXPLAIN output captured from TiDB v8.5.0.
+func TestGetAffectedRowsFromPlan(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ plan []planRow
+ want int64
+ wantErr bool
+ }{
+ {
+ name: "join_estimates_joined_rows_not_its_inputs",
+ plan: []planRow{
+ {"Update_8", "N/A"},
+ {"└─HashJoin_27", "800.00"},
+ {" ├─TableReader_44(Build)", "0.40"},
+ {" │ └─Selection_43", "0.40"},
+ {" │ └─TableFullScan_42", "400.00"},
+ {" └─TableReader_38(Probe)", "1000.00"},
+ {" └─Selection_37", "1000.00"},
+ {" └─TableFullScan_36", "1000.00"},
+ },
+ want: 800,
+ },
+ {
+ name: "limit_is_part_of_the_plan",
+ plan: []planRow{
+ {"Update_6", "N/A"},
+ {"└─TopN_9", "20.00"},
+ {" └─TableReader_17", "20.00"},
+ {" └─TopN_16", "20.00"},
+ {" └─Selection_15", "500.00"},
+ {" └─TableFullScan_14", "1000.00"},
+ },
+ want: 20,
+ },
+ {
+ name: "foreign_key_cascade_follows_the_input",
+ plan: []planRow{
+ {"Delete_3", "N/A"},
+ {"├─TableReader_8", "9.00"},
+ {"│ └─TableRangeScan_7", "9.00"},
+ {"└─Foreign_Key_Cascade_4", "0.00"},
+ },
+ want: 9,
+ },
+ {
+ name: "insert_set_has_no_input",
+ plan: []planRow{
+ {"Insert_1", "N/A"},
+ },
+ want: 0,
+ },
+ {
+ // With tidb_opt_enable_non_eval_scalar_subquery, a subquery in SET is
+ // planned as a second root rather than evaluated during planning.
+ name: "subquery_root_is_not_an_input",
+ plan: []planRow{
+ {"Insert_1", "N/A"},
+ {"ScalarSubQuery_36", "N/A"},
+ {"└─MaxOneRow_11", "1.00"},
+ {" └─Projection_12", "1.00"},
+ {" └─StreamAgg_14", "1.00"},
+ {" └─Limit_30", "1.00"},
+ {" └─TableReader_35", "1.00"},
+ {" └─Limit_34", "1.00"},
+ {" └─TableFullScan_24", "1.00"},
+ },
+ want: 0,
+ },
+ {
+ name: "select_root_is_rejected",
+ plan: []planRow{
+ {"TableReader_7", "100.00"},
+ {"└─Selection_6", "100.00"},
+ {" └─TableFullScan_5", "1000.00"},
+ },
+ wantErr: true,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := getAffectedRowsFromPlan(tc.plan)
+ if tc.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ require.Equal(t, tc.want, common.RoundRows(got))
+ })
+ }
+}
+
+func TestGetAffectedRows(t *testing.T) {
+ plan := []planRow{
+ {"Delete_7", "N/A"},
+ {"└─HashJoin_20", "0.40"},
+ }
+ // 0.4 joined rows change a row in each of two tables.
+ got, err := getAffectedRows(plan, "DELETE a, b FROM a JOIN b ON a.id = b.id")
+ require.NoError(t, err)
+ require.Equal(t, int64(1), got)
+}
+
+func TestCountDMLTargets(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ want int
+ }{
+ {statement: "UPDATE t SET a = 1, b = 2 WHERE id = 1", want: 1},
+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.c = 1, a.d = 2", want: 1},
+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.c = 1, b.c = 1", want: 2},
+ {statement: "UPDATE a AS x, b AS y SET x.c = 1, y.c = 1 WHERE x.id = y.id", want: 2},
+ // Either joined table may own each unqualified column, up to the two joined tables.
+ {statement: "UPDATE a JOIN b ON a.id = b.id SET c = 1, d = 2, e = 3", want: 2},
+ {statement: "DELETE a, b FROM a JOIN b ON a.id = b.id", want: 2},
+ {statement: "DELETE a FROM a JOIN b ON a.id = b.id", want: 1},
+ {statement: "DELETE FROM t WHERE id = 1", want: 1},
+ {statement: "INSERT INTO t SELECT * FROM s", want: 1},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ require.Equal(t, tc.want, countDMLTargets(tc.statement))
+ })
+ }
+}
diff --git a/backend/plugin/db/tidb/testutil_test.go b/backend/plugin/db/tidb/testutil_test.go
new file mode 100644
index 0000000000..942f7e2dae
--- /dev/null
+++ b/backend/plugin/db/tidb/testutil_test.go
@@ -0,0 +1,32 @@
+package tidb
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+func openTestDriver(ctx context.Context, t *testing.T, container *testcontainer.Container) *Driver {
+ t.Helper()
+
+ driver := &Driver{}
+ d, err := driver.Open(ctx, storepb.Engine_TIDB, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Username: "root",
+ Host: container.GetHost(),
+ Port: container.GetPort(),
+ },
+ ConnectionContext: db.ConnectionContext{},
+ })
+ require.NoError(t, err)
+
+ tidbDriver, ok := d.(*Driver)
+ require.True(t, ok)
+ return tidbDriver
+}
diff --git a/backend/plugin/db/tidb/tidb_test.go b/backend/plugin/db/tidb/tidb_test.go
index 9d07428224..20c53ad3f7 100644
--- a/backend/plugin/db/tidb/tidb_test.go
+++ b/backend/plugin/db/tidb/tidb_test.go
@@ -92,26 +92,6 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T
require.Equal(t, statement, commands[0].Text)
}
-func openTestDriver(ctx context.Context, t *testing.T, container *testcontainer.Container) *Driver {
- t.Helper()
-
- driver := &Driver{}
- d, err := driver.Open(ctx, storepb.Engine_TIDB, db.ConnectionConfig{
- DataSource: &storepb.DataSource{
- Type: storepb.DataSourceType_ADMIN,
- Username: "root",
- Host: container.GetHost(),
- Port: container.GetPort(),
- },
- ConnectionContext: db.ConnectionContext{},
- })
- require.NoError(t, err)
-
- tidbDriver, ok := d.(*Driver)
- require.True(t, ok)
- return tidbDriver
-}
-
func TestExecuteCreateIndexInTransaction(t *testing.T) {
t.Parallel()
ctx := context.Background()
diff --git a/backend/plugin/parser/base/interface.go b/backend/plugin/parser/base/interface.go
index 0b0a74d4f2..9e7a35a029 100644
--- a/backend/plugin/parser/base/interface.go
+++ b/backend/plugin/parser/base/interface.go
@@ -393,9 +393,39 @@ func isAllDMLImpl(engine storepb.Engine, statement string) bool {
type ChangeSummary struct {
ChangedResources *model.ChangedResources
- SampleDMLS []string
- DMLCount int
- InsertCount int
+ // DMLStatements holds the text of every DML statement whose rows can be estimated, in order.
+ // A statement that runs after the sheet changes the search path starts with WithSearchPath's
+ // prefix, which the driver splits off with SplitSearchPath and runs before the EXPLAIN.
+ DMLStatements []string
+ // DMLCount counts the DML statements, including those DMLStatements omits.
+ DMLCount int
+ // InsertCount counts the rows of INSERT ... VALUES statements.
+ InsertCount int
+}
+
+// searchPathPrefix starts the statement WithSearchPath puts before a DML statement.
+const searchPathPrefix = "SET LOCAL search_path TO "
+
+// WithSearchPath returns statement preceded by a SET LOCAL search_path statement for schemas.
+func WithSearchPath(statement string, schemas []string) string {
+ quoted := make([]string, len(schemas))
+ for i, schema := range schemas {
+ quoted[i] = `"` + strings.ReplaceAll(schema, `"`, `""`) + `"`
+ }
+ return searchPathPrefix + strings.Join(quoted, ", ") + ";\n" + statement
+}
+
+// SplitSearchPath returns the SET LOCAL search_path statement that WithSearchPath put before
+// statement, or "" when there is none, and the statement after it.
+func SplitSearchPath(statement string) (string, string) {
+ if !strings.HasPrefix(statement, searchPathPrefix) {
+ return "", statement
+ }
+ setup, rest, ok := strings.Cut(statement, ";\n")
+ if !ok {
+ return "", statement
+ }
+ return setup, rest
}
// REFACTOR(zp): Put it here to avoid circular import for now.
diff --git a/backend/plugin/parser/base/interface_test.go b/backend/plugin/parser/base/interface_test.go
index 1001937c76..660dd03382 100644
--- a/backend/plugin/parser/base/interface_test.go
+++ b/backend/plugin/parser/base/interface_test.go
@@ -1,6 +1,23 @@
package base
-import "testing"
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestWithSearchPath(t *testing.T) {
+ statement := WithSearchPath("DELETE FROM t;", []string{"app", `we"ird`})
+ require.Equal(t, "SET LOCAL search_path TO \"app\", \"we\"\"ird\";\nDELETE FROM t;", statement)
+
+ setup, rest := SplitSearchPath(statement)
+ require.Equal(t, `SET LOCAL search_path TO "app", "we""ird"`, setup)
+ require.Equal(t, "DELETE FROM t;", rest)
+
+ setup, rest = SplitSearchPath("DELETE FROM t;")
+ require.Empty(t, setup)
+ require.Equal(t, "DELETE FROM t;", rest)
+}
func TestTSQLRecognizeExplainType(t *testing.T) {
testCases := []struct {
diff --git a/backend/plugin/parser/cockroachdb/resource_change.go b/backend/plugin/parser/cockroachdb/resource_change.go
new file mode 100644
index 0000000000..e850bf00f2
--- /dev/null
+++ b/backend/plugin/parser/cockroachdb/resource_change.go
@@ -0,0 +1,477 @@
+package cockroachdb
+
+import (
+ "slices"
+ "strings"
+
+ "github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+func init() {
+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_COCKROACHDB, extractChangedResources)
+}
+
+// extractChangedResources resolves an unqualified table to the current database and to a schema of
+// the search path: currentSchema, or else the search path synced in dbMetadata, or "public", until
+// the statements SET another. Like the PostgreSQL extractor, an existing table resolves to the first
+// schema that has it in dbMetadata, and a new table to the first schema.
+func extractChangedResources(database string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
+ defaultSearchPath := []string{currentSchema}
+ if currentSchema == "" {
+ defaultSearchPath = []string{"public"}
+ if dbMetadata != nil && len(dbMetadata.GetSearchPath()) > 0 {
+ defaultSearchPath = dbMetadata.GetSearchPath()
+ }
+ }
+ searchPath := defaultSearchPath
+ // sessionSearchPath leaves out SET LOCAL, which lasts until the transaction ends.
+ sessionSearchPath := searchPath
+ summary := &base.ChangeSummary{
+ ChangedResources: model.NewChangedResources(dbMetadata),
+ }
+ schemaOf := func(table string) string {
+ if dbMetadata == nil {
+ return searchPath[0]
+ }
+ for _, schemaName := range searchPath {
+ schema := dbMetadata.GetSchemaMetadata(schemaName)
+ if schema != nil && (schema.GetTable(table) != nil || schema.GetView(table) != nil || schema.GetMaterializedView(table) != nil) {
+ return schemaName
+ }
+ }
+ return searchPath[0]
+ }
+ addTable := func(name *tree.TableName, affectedTable bool) {
+ db, schema, table := resolveTableName(name, database, schemaOf(name.Table()))
+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, affectedTable)
+ }
+ // addIndexTable records the table of an index, which dbMetadata supplies when the name omits it.
+ addIndexTable := func(index *tree.TableIndexName) {
+ if index.Table.ObjectName != "" {
+ addTable(&index.Table, false)
+ return
+ }
+ // dbMetadata describes only the current database, so an index of another is recorded by its database.
+ if index.Table.ExplicitCatalog && index.Table.Catalog() != database {
+ summary.ChangedResources.AddDatabase(index.Table.Catalog())
+ return
+ }
+ if dbMetadata == nil {
+ return
+ }
+ schemas := searchPath
+ if index.Table.ExplicitSchema {
+ schemas = []string{index.Table.Schema()}
+ }
+ schema, indexMetadata := dbMetadata.SearchIndex(schemas, string(index.Index))
+ if indexMetadata == nil {
+ return
+ }
+ db, _, _ := resolveTableName(&index.Table, database, schema)
+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: indexMetadata.GetTableProto().GetName()}, false)
+ }
+ // addMutations records the target of each mutation in the statement and returns their number.
+ addMutations := func(stmt tree.Statement) int {
+ targets := getMutationTargets(stmt)
+ for _, target := range targets {
+ if name, ok := getTableName(target); ok {
+ addTable(name, false)
+ }
+ }
+ return len(targets)
+ }
+ addDML := func(text string) {
+ summary.DMLCount++
+ text = strings.TrimSpace(text)
+ // The EXPLAIN connection starts with the default search path, so a changed one is replayed.
+ if !slices.Equal(searchPath, defaultSearchPath) {
+ text = base.WithSearchPath(text, searchPath)
+ }
+ summary.DMLStatements = append(summary.DMLStatements, text)
+ }
+
+ for _, ast := range asts {
+ crdbAST, ok := ast.(*AST)
+ if !ok {
+ return nil, errors.New("expected CockroachDB AST")
+ }
+ stmt, text := crdbAST.Stmt.AST, crdbAST.Stmt.SQL
+ // EXPLAIN ANALYZE executes the statement it explains.
+ if explain, ok := stmt.(*tree.ExplainAnalyze); ok {
+ stmt, text = explain.Statement, tree.AsString(explain.Statement)
+ }
+ switch n := stmt.(type) {
+ case *tree.Insert:
+ mutationCount := addMutations(n)
+ // Nested mutations make the whole statement a sample, whose estimate includes the VALUES rows.
+ if rows, ok := getInsertValuesRowCount(n); ok && mutationCount == 1 {
+ summary.InsertCount += rows
+ } else {
+ addDML(text)
+ }
+ case *tree.Update, *tree.Delete:
+ addMutations(n)
+ addDML(text)
+ case *tree.Select:
+ if addMutations(n) > 0 {
+ addDML(text)
+ }
+ case *tree.CreateTable:
+ db, schema, table := resolveTableName(&n.Table, database, searchPath[0])
+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
+ if n.AsSource != nil && addMutations(n.AsSource) > 0 {
+ addDML(text)
+ }
+ case *tree.CreateView:
+ if n.Materialized {
+ db, schema, table := resolveTableName(&n.Name, database, searchPath[0])
+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ case *tree.Import:
+ // The plan cannot estimate the rows an import loads, so, as for LOAD DATA in the MySQL family,
+ // only its target is recorded.
+ if n.Into && n.Table != nil {
+ addTable(n.Table, false)
+ }
+ case *tree.AlterTable:
+ name := n.Table.ToTableName()
+ addTable(&name, true)
+ case *tree.AlterTableLocality:
+ name := n.Name.ToTableName()
+ addTable(&name, true)
+ case *tree.AlterTableSetSchema:
+ if !n.IsSequence && (!n.IsView || n.IsMaterialized) {
+ name := n.Name.ToTableName()
+ addTable(&name, true)
+ }
+ case *tree.AlterTableOwner:
+ if !n.IsSequence && (!n.IsView || n.IsMaterialized) {
+ name := n.Name.ToTableName()
+ addTable(&name, true)
+ }
+ case *tree.DropTable:
+ for i := range n.Names {
+ addTable(&n.Names[i], true)
+ }
+ case *tree.DropView:
+ // A materialized view holds data, so its drop is recorded like a table's.
+ if n.IsMaterialized {
+ for i := range n.Names {
+ addTable(&n.Names[i], true)
+ }
+ }
+ case *tree.Truncate:
+ for i := range n.Tables {
+ addTable(&n.Tables[i], true)
+ }
+ case *tree.RenameTable:
+ if n.IsSequence || (n.IsView && !n.IsMaterialized) {
+ continue
+ }
+ name := n.Name.ToTableName()
+ db, schema, table := resolveTableName(&name, database, schemaOf(name.Table()))
+ summary.ChangedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, true)
+ // An unqualified new name stays in the schema of the renamed table.
+ newName := n.NewName.ToTableName()
+ newDB, newSchema, newTable := resolveTableName(&newName, db, schema)
+ summary.ChangedResources.AddTable(newDB, newSchema, &storepb.ChangedResourceTable{Name: newTable}, false)
+ case *tree.CreateIndex:
+ addTable(&n.Table, false)
+ case *tree.DropIndex:
+ for _, index := range n.IndexList {
+ addIndexTable(index)
+ }
+ case *tree.AlterIndex:
+ addIndexTable(&n.Index)
+ case *tree.AlterIndexVisible:
+ addIndexTable(&n.Index)
+ case *tree.RenameIndex:
+ addIndexTable(n.Index)
+ case *tree.SetZoneConfig:
+ switch {
+ case n.TableOrIndex.Index != "":
+ addIndexTable(&n.TableOrIndex)
+ case n.TableOrIndex.Table.ObjectName != "":
+ addTable(&n.TableOrIndex.Table, true)
+ default:
+ }
+ case *tree.Discard:
+ if n.Mode == tree.DiscardModeAll {
+ searchPath, sessionSearchPath = defaultSearchPath, defaultSearchPath
+ }
+ case *tree.CommitTransaction, *tree.RollbackTransaction:
+ // Unlike PostgreSQL, CockroachDB keeps a SET made in a transaction that rolls back.
+ searchPath = sessionSearchPath
+ case *tree.SetVar:
+ if n.ResetAll || strings.EqualFold(n.Name, "search_path") {
+ searchPath = getSearchPath(n, defaultSearchPath)
+ if !n.Local {
+ sessionSearchPath = searchPath
+ }
+ }
+ default:
+ }
+ }
+ return summary, nil
+}
+
+// getSearchPath returns the schemas a SET search_path names, skipping "$user", or defaultSearchPath
+// when it names none or resets the search path.
+func getSearchPath(set *tree.SetVar, defaultSearchPath []string) []string {
+ if set.ResetAll {
+ return defaultSearchPath
+ }
+ var searchPath []string
+ for _, value := range set.Values {
+ var schema string
+ switch v := value.(type) {
+ case *tree.UnresolvedName:
+ schema = v.Parts[0]
+ case *tree.StrVal:
+ schema = v.RawString()
+ default:
+ return defaultSearchPath
+ }
+ if schema != "$user" {
+ searchPath = append(searchPath, schema)
+ }
+ }
+ if len(searchPath) == 0 {
+ return defaultSearchPath
+ }
+ return searchPath
+}
+
+func resolveTableName(name *tree.TableName, defaultDatabase string, defaultSchema string) (string, string, string) {
+ database, schema := defaultDatabase, defaultSchema
+ if name.ExplicitCatalog {
+ database = name.Catalog()
+ }
+ if name.ExplicitSchema {
+ schema = name.Schema()
+ }
+ return database, schema, name.Table()
+}
+
+// getTableName returns the table name of a mutation target, which is false for a numeric table
+// reference.
+func getTableName(target tree.TableExpr) (*tree.TableName, bool) {
+ if aliased, ok := target.(*tree.AliasedTableExpr); ok {
+ target = aliased.Expr
+ }
+ name, ok := target.(*tree.TableName)
+ return name, ok
+}
+
+// getInsertValuesRowCount returns the rows of INSERT ... VALUES and INSERT ... DEFAULT VALUES; ok is
+// false when a query supplies the rows.
+func getInsertValuesRowCount(insert *tree.Insert) (int, bool) {
+ if insert.Rows == nil || insert.Rows.Select == nil {
+ return 1, true
+ }
+ rows := insert.Rows.Select
+ for {
+ paren, ok := rows.(*tree.ParenSelect)
+ if !ok || paren.Select == nil {
+ break
+ }
+ rows = paren.Select.Select
+ }
+ if values, ok := rows.(*tree.ValuesClause); ok {
+ return len(values.Rows), true
+ }
+ return 0, false
+}
+
+// getMutationTargets returns the target of every INSERT, UPSERT, UPDATE, and DELETE in stmt: stmt
+// itself, its CTEs, and statement sources such as [DELETE FROM t RETURNING id] at any depth.
+func getMutationTargets(stmt tree.Statement) []tree.TableExpr {
+ c := &mutationCollector{}
+ c.statement(stmt)
+ return c.targets
+}
+
+// getMutationTypes returns the statement type of each mutation getMutationTargets finds, in order.
+func getMutationTypes(stmt tree.Statement) []storepb.StatementType {
+ c := &mutationCollector{}
+ c.statement(stmt)
+ return c.types
+}
+
+type mutationCollector struct {
+ targets []tree.TableExpr
+ types []storepb.StatementType
+}
+
+func (c *mutationCollector) statement(stmt tree.Statement) {
+ switch n := stmt.(type) {
+ case *tree.Insert:
+ c.targets = append(c.targets, n.Table)
+ c.types = append(c.types, storepb.StatementType_INSERT)
+ c.with(n.With)
+ c.selectQuery(n.Rows)
+ if n.OnConflict != nil {
+ c.updateExprs(n.OnConflict.Exprs)
+ c.where(n.OnConflict.Where)
+ c.expr(n.OnConflict.ArbiterPredicate)
+ }
+ c.returning(n.Returning)
+ case *tree.Update:
+ c.targets = append(c.targets, n.Table)
+ c.types = append(c.types, storepb.StatementType_UPDATE)
+ c.with(n.With)
+ c.updateExprs(n.Exprs)
+ c.tableExprs(n.From)
+ c.where(n.Where)
+ c.orderBy(n.OrderBy)
+ c.limit(n.Limit)
+ c.returning(n.Returning)
+ case *tree.Delete:
+ c.targets = append(c.targets, n.Table)
+ c.types = append(c.types, storepb.StatementType_DELETE)
+ c.with(n.With)
+ c.tableExprs(n.Using)
+ c.where(n.Where)
+ c.orderBy(n.OrderBy)
+ c.limit(n.Limit)
+ c.returning(n.Returning)
+ case *tree.Select:
+ c.selectQuery(n)
+ case tree.SelectStatement:
+ c.selectStatement(n)
+ default:
+ }
+}
+
+func (c *mutationCollector) selectQuery(sel *tree.Select) {
+ if sel == nil {
+ return
+ }
+ c.with(sel.With)
+ c.selectStatement(sel.Select)
+ c.orderBy(sel.OrderBy)
+ c.limit(sel.Limit)
+}
+
+func (c *mutationCollector) selectStatement(stmt tree.SelectStatement) {
+ switch n := stmt.(type) {
+ case *tree.ParenSelect:
+ c.selectQuery(n.Select)
+ case *tree.SelectClause:
+ for _, selectExpr := range n.Exprs {
+ c.expr(selectExpr.Expr)
+ }
+ c.tableExprs(n.From.Tables)
+ c.where(n.Where)
+ c.exprs(n.GroupBy)
+ c.where(n.Having)
+ c.exprs(n.DistinctOn)
+ for _, window := range n.Window {
+ c.exprs(window.Partitions)
+ c.orderBy(window.OrderBy)
+ }
+ case *tree.UnionClause:
+ c.selectQuery(n.Left)
+ c.selectQuery(n.Right)
+ case *tree.ValuesClause:
+ for _, row := range n.Rows {
+ c.exprs(row)
+ }
+ default:
+ }
+}
+
+func (c *mutationCollector) with(with *tree.With) {
+ if with == nil {
+ return
+ }
+ for _, cte := range with.CTEList {
+ c.statement(cte.Stmt)
+ }
+}
+
+func (c *mutationCollector) tableExprs(exprs tree.TableExprs) {
+ for _, expr := range exprs {
+ c.tableExpr(expr)
+ }
+}
+
+func (c *mutationCollector) tableExpr(expr tree.TableExpr) {
+ switch n := expr.(type) {
+ case *tree.AliasedTableExpr:
+ c.tableExpr(n.Expr)
+ case *tree.ParenTableExpr:
+ c.tableExpr(n.Expr)
+ case *tree.JoinTableExpr:
+ c.tableExpr(n.Left)
+ c.tableExpr(n.Right)
+ if cond, ok := n.Cond.(*tree.OnJoinCond); ok {
+ c.expr(cond.Expr)
+ }
+ case *tree.RowsFromExpr:
+ c.exprs(n.Items)
+ case *tree.Subquery:
+ c.selectStatement(n.Select)
+ case *tree.StatementSource:
+ c.statement(n.Statement)
+ default:
+ }
+}
+
+func (c *mutationCollector) expr(expr tree.Expr) {
+ if expr == nil {
+ return
+ }
+ // The visit function never fails, so neither does the visit.
+ _, _ = tree.SimpleVisit(expr, func(e tree.Expr) (bool, tree.Expr, error) {
+ if subquery, ok := e.(*tree.Subquery); ok {
+ c.selectStatement(subquery.Select)
+ return false, e, nil
+ }
+ return true, e, nil
+ })
+}
+
+func (c *mutationCollector) exprs(exprs []tree.Expr) {
+ for _, expr := range exprs {
+ c.expr(expr)
+ }
+}
+
+func (c *mutationCollector) where(where *tree.Where) {
+ if where != nil {
+ c.expr(where.Expr)
+ }
+}
+
+func (c *mutationCollector) orderBy(orderBy tree.OrderBy) {
+ for _, order := range orderBy {
+ c.expr(order.Expr)
+ }
+}
+
+func (c *mutationCollector) limit(limit *tree.Limit) {
+ if limit != nil {
+ c.expr(limit.Count)
+ c.expr(limit.Offset)
+ }
+}
+
+func (c *mutationCollector) updateExprs(exprs tree.UpdateExprs) {
+ for _, updateExpr := range exprs {
+ c.expr(updateExpr.Expr)
+ }
+}
+
+func (c *mutationCollector) returning(returning tree.ReturningClause) {
+ if exprs, ok := returning.(*tree.ReturningExprs); ok {
+ for _, selectExpr := range *exprs {
+ c.expr(selectExpr.Expr)
+ }
+ }
+}
diff --git a/backend/plugin/parser/cockroachdb/resource_change_test.go b/backend/plugin/parser/cockroachdb/resource_change_test.go
new file mode 100644
index 0000000000..cf18e447ed
--- /dev/null
+++ b/backend/plugin/parser/cockroachdb/resource_change_test.go
@@ -0,0 +1,401 @@
+package cockroachdb
+
+import (
+ "fmt"
+ "slices"
+ "strings"
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+
+ // Registers the statement splitter that CockroachDB parsing uses.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/pg"
+)
+
+func TestExtractChangedResources(t *testing.T) {
+ const statement = `CREATE TABLE t1 (c1 INT);
+DROP TABLE t1;
+ALTER TABLE t1 ADD COLUMN c2 INT;
+ALTER TABLE t1 RENAME TO t2;
+INSERT INTO t1 (c1) VALUES (1), (5);
+UPDATE t1
+SET c1 = 5
+WHERE c1 = 1;
+DELETE FROM t1; -- trailing comment
+`
+ want := model.NewChangedResources(nil /* dbMetadata */)
+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t1"}, true)
+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t2"}, false)
+
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, &base.ChangeSummary{
+ ChangedResources: want,
+ DMLCount: 2,
+ DMLStatements: []string{"UPDATE t1\nSET c1 = 5\nWHERE c1 = 1", "DELETE FROM t1"},
+ InsertCount: 2,
+ }, got)
+}
+
+func TestExtractChangedResourcesDMLCounts(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ wantDMLCount int
+ wantInsertCount int
+ wantTables []string
+ }{
+ {
+ name: "insert values",
+ statement: `INSERT INTO t VALUES (1), (2), (3)`,
+ wantInsertCount: 3,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "parenthesized insert values",
+ statement: `INSERT INTO t (VALUES (1), (2))`,
+ wantInsertCount: 2,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "upsert values",
+ statement: `UPSERT INTO t VALUES (1), (2)`,
+ wantInsertCount: 2,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert values on conflict",
+ statement: `INSERT INTO t VALUES (1), (2) ON CONFLICT (id) DO UPDATE SET c = excluded.c`,
+ wantInsertCount: 2,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert default values",
+ statement: `INSERT INTO t DEFAULT VALUES`,
+ wantInsertCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert select",
+ statement: `INSERT INTO t SELECT * FROM t2 ON CONFLICT DO NOTHING`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "upsert select",
+ statement: `UPSERT INTO t SELECT * FROM t2`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert select with read-only CTE",
+ statement: `WITH x AS (SELECT id FROM t2) INSERT INTO t SELECT id FROM x`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "update",
+ statement: `UPDATE t SET c = 1 FROM t2 WHERE t.id = t2.id`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "delete",
+ statement: `DELETE FROM t USING t2 WHERE t.id = t2.id ORDER BY t.id LIMIT 10`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "data-modifying CTE feeding insert",
+ statement: `WITH d AS (DELETE FROM t WHERE id > 0 RETURNING *) INSERT INTO t2 SELECT * FROM d`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "data-modifying CTE feeding insert values",
+ statement: `WITH i AS (INSERT INTO t VALUES (1) RETURNING id) INSERT INTO t2 VALUES (2), (3)`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "data-modifying CTE in select",
+ statement: `WITH d AS (DELETE FROM t RETURNING 1) SELECT count(*) FROM d`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "data-modifying CTEs feeding delete",
+ statement: `WITH u AS (UPDATE t SET c = 1 RETURNING id), i AS (UPSERT INTO t3 SELECT * FROM t2 RETURNING id) DELETE FROM t2 WHERE id IN (SELECT id FROM u)`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2", "t3"},
+ },
+ {
+ name: "statement source in select",
+ statement: `SELECT * FROM [DELETE FROM t WHERE c < 10 RETURNING id]`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "statement source in scalar subquery of insert values",
+ statement: `INSERT INTO t2 VALUES ((SELECT max(id) FROM [DELETE FROM t RETURNING id]))`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "statement source in update subquery",
+ statement: `UPDATE t SET c = 1 WHERE id IN (SELECT id FROM [DELETE FROM t2 WHERE c < 3 RETURNING id])`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "statement source joined in delete using",
+ statement: `DELETE FROM t USING t3 JOIN [UPDATE t2 SET c = 1 RETURNING id] AS u ON t3.id = u.id WHERE t.id = t3.id`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "statement source in create table as",
+ statement: `CREATE TABLE t_moved AS SELECT * FROM [DELETE FROM t RETURNING *]`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t_moved"},
+ },
+ {
+ name: "create table as",
+ statement: `CREATE TABLE t_copy AS SELECT * FROM t`,
+ wantTables: []string{"t_copy"},
+ },
+ {
+ name: "read-only CTE in select",
+ statement: `WITH x AS (SELECT id FROM t) SELECT * FROM x WHERE id IN (SELECT id FROM [SHOW TABLES])`,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ want := model.NewChangedResources(nil /* dbMetadata */)
+ for _, table := range tc.wantTables {
+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ var wantDMLStatements []string
+ if tc.wantDMLCount > 0 {
+ wantDMLStatements = []string{tc.statement}
+ }
+
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, tc.statement), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, &base.ChangeSummary{
+ ChangedResources: want,
+ DMLCount: tc.wantDMLCount,
+ DMLStatements: wantDMLStatements,
+ InsertCount: tc.wantInsertCount,
+ }, got)
+ })
+ }
+}
+
+func TestExtractChangedResourcesFollowsSearchPath(t *testing.T) {
+ const statement = `DELETE FROM t1;
+SET search_path = app, public;
+DELETE FROM t2;
+SET search_path TO '$user', "Reports";
+DELETE FROM t3;
+RESET search_path;
+DELETE FROM t4;`
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{"db.Reports.t3", "db.app.t2", "db.public.t1", "db.public.t4"}, getTableNames(got.ChangedResources))
+ // A statement after the search path changes carries it for its EXPLAIN.
+ require.Equal(t, []string{
+ "DELETE FROM t1",
+ "SET LOCAL search_path TO \"app\", \"public\";\nDELETE FROM t2",
+ "SET LOCAL search_path TO \"Reports\";\nDELETE FROM t3",
+ "DELETE FROM t4",
+ }, got.DMLStatements)
+
+ t.Run("SET LOCAL lasts until the transaction ends, SET outlasts ROLLBACK, and DISCARD ALL resets", func(t *testing.T) {
+ const statement = `BEGIN;
+SET LOCAL search_path = app;
+DELETE FROM t1;
+COMMIT;
+DELETE FROM t2;
+BEGIN;
+SET search_path = app;
+SET LOCAL search_path = other;
+ROLLBACK;
+DELETE FROM t3;
+DISCARD ALL;
+DELETE FROM t4;`
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{"db.app.t1", "db.app.t3", "db.public.t2", "db.public.t4"}, getTableNames(got.ChangedResources))
+ })
+
+ t.Run("the synced search path applies without a current schema", func(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ SearchPath: `"$user", app, public`,
+ Schemas: []*metadatapb.SchemaMetadata{
+ {Name: "app", Tables: []*metadatapb.TableMetadata{{Name: "t"}}},
+ {Name: "public", Tables: []*metadatapb.TableMetadata{{Name: "t2"}}},
+ },
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)
+ const statement = `DELETE FROM t; DELETE FROM t2; DROP MATERIALIZED VIEW mv; DROP VIEW v;`
+ got, err := extractChangedResources("db", "", dbMetadata, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{"db.app.mv", "db.app.t", "db.public.t2"}, getTableNames(got.ChangedResources))
+ })
+
+ t.Run("existing tables resolve to the first schema that has them", func(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {Name: "public", Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 1000}}},
+ {Name: "empty"},
+ },
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)
+ const statement = `SET search_path = empty, public;
+DELETE FROM t;
+CREATE TABLE t2 (id INT PRIMARY KEY);`
+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{"db.empty.t2", "db.public.t"}, getTableNames(got.ChangedResources))
+ })
+}
+
+func TestExtractChangedResourcesNames(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ currentSchema string
+ want string
+ }{
+ {name: "unqualified defaults to public", statement: `UPDATE t SET c = 1`, want: "db.public.t"},
+ {name: "unqualified uses the current schema", statement: `UPDATE t SET c = 1`, currentSchema: "app", want: "db.app.t"},
+ {name: "schema qualified with alias", statement: `INSERT INTO myschema.t AS x VALUES (1)`, currentSchema: "app", want: "db.myschema.t"},
+ {name: "database qualified", statement: `DELETE FROM otherdb.myschema.t`, want: "otherdb.myschema.t"},
+ {name: "quoted identifiers keep their case", statement: `UPDATE "MySchema"."T" SET c = 1`, want: "db.MySchema.T"},
+ {name: "unquoted identifiers are lowercased", statement: `TRUNCATE MySchema.T`, want: "db.myschema.t"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := extractChangedResources("db", tc.currentSchema, nil /* dbMetadata */, parseASTs(t, tc.statement), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{tc.want}, getTableNames(got.ChangedResources))
+ })
+ }
+}
+
+func TestExtractChangedResourcesDDL(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {Name: "t", RowCount: 10},
+ {Name: "t2", RowCount: 1000, Indexes: []*metadatapb.IndexMetadata{{Name: "t2_c_idx"}}},
+ },
+ },
+ {
+ Name: "app",
+ Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 100_000, Indexes: []*metadatapb.IndexMetadata{{Name: "t_c_idx"}}}},
+ },
+ },
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_COCKROACHDB, true /* caseSensitive */)
+ for _, tc := range []struct {
+ statement string
+ wantTables []string
+ wantAffectedRows int64
+ }{
+ {statement: `CREATE TABLE t3 (id INT PRIMARY KEY)`, wantTables: []string{"db.public.t3"}},
+ {statement: `CREATE INDEX idx ON t2 (c)`, wantTables: []string{"db.public.t2"}},
+ {statement: `CREATE MATERIALIZED VIEW mv AS SELECT * FROM t`, wantTables: []string{"db.public.mv"}},
+ {statement: `CREATE VIEW v AS SELECT * FROM t`},
+ {statement: `IMPORT INTO t2 (c) CSV DATA ('nodelocal://1/t2.csv')`, wantTables: []string{"db.public.t2"}},
+ {statement: `ALTER TABLE t2 CONFIGURE ZONE USING num_replicas = 5`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER INDEX t2_c_idx CONFIGURE ZONE USING num_replicas = 5`, wantTables: []string{"db.public.t2"}},
+ {statement: `ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5`},
+ {statement: `DROP INDEX t2@t2_c_idx, t2_c_idx, app.t_c_idx, missing_idx`, wantTables: []string{"db.app.t", "db.public.t2"}},
+ {statement: `DROP INDEX db.public.t2_c_idx`, wantTables: []string{"db.public.t2"}},
+ {statement: `ALTER INDEX app.t@t_c_idx PARTITION BY NOTHING`, wantTables: []string{"db.app.t"}},
+ {statement: `ALTER INDEX t2_c_idx NOT VISIBLE`, wantTables: []string{"db.public.t2"}},
+ {statement: `ALTER INDEX t2_c_idx RENAME TO t2_idx`, wantTables: []string{"db.public.t2"}},
+ {statement: `ALTER TABLE t ADD COLUMN c2 INT, DROP COLUMN c`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE t RENAME COLUMN c TO c2`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE t SET LOCALITY REGIONAL BY ROW`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE t SET SCHEMA app`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE t OWNER TO u`, wantTables: []string{"db.public.t"}, wantAffectedRows: 10},
+ {statement: `ALTER VIEW t2 SET SCHEMA app`},
+ {statement: `ALTER MATERIALIZED VIEW t2 SET SCHEMA app`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER MATERIALIZED VIEW t2 OWNER TO u`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER TABLE t RENAME TO t_new`, wantTables: []string{"db.public.t", "db.public.t_new"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE app.t RENAME TO t_new`, wantTables: []string{"db.app.t", "db.app.t_new"}, wantAffectedRows: 100_000},
+ {statement: `ALTER MATERIALIZED VIEW t2 RENAME TO mv`, wantTables: []string{"db.public.mv", "db.public.t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER VIEW t2 RENAME TO v`},
+ {statement: `ALTER SEQUENCE t2 RENAME TO s`},
+ {statement: `DROP TABLE t, app.t`, wantTables: []string{"db.app.t", "db.public.t"}, wantAffectedRows: 100_010},
+ {statement: `TRUNCATE t2`, wantTables: []string{"db.public.t2"}, wantAffectedRows: 1000},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, tc.statement), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, tc.wantTables, getTableNames(got.ChangedResources))
+ require.Equal(t, tc.wantAffectedRows, got.ChangedResources.CountAffectedTableRows())
+ require.Zero(t, got.DMLCount)
+ })
+ }
+
+ t.Run("an index of another database is recorded by its database", func(t *testing.T) {
+ const statement = `DROP INDEX otherdb.public.t2_c_idx`
+ got, err := extractChangedResources("db", "public", dbMetadata, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Empty(t, getTableNames(got.ChangedResources))
+ require.Equal(t, []string{"otherdb"}, got.ChangedResources.GetDatabaseOnlyTargets())
+ })
+}
+
+func TestExtractChangedResourcesExplainAnalyze(t *testing.T) {
+ const statement = `EXPLAIN ANALYZE (VERBOSE) DELETE FROM t WHERE id = 1;
+EXPLAIN ANALYZE UPSERT INTO t2 SELECT * FROM t;
+EXPLAIN UPDATE t3 SET c = 1;`
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{"db.public.t", "db.public.t2"}, getTableNames(got.ChangedResources))
+ require.Equal(t, []string{"DELETE FROM t WHERE id = 1", "UPSERT INTO t2 SELECT * FROM t"}, got.DMLStatements)
+}
+
+func TestExtractChangedResourcesListsEveryDMLStatement(t *testing.T) {
+ var statements []string
+ for i := range 12 {
+ statements = append(statements, fmt.Sprintf("DELETE FROM t WHERE id = %d", i))
+ }
+ statement := strings.Join(statements, ";\n")
+
+ got, err := extractChangedResources("db", "public", nil /* dbMetadata */, parseASTs(t, statement), statement)
+ require.NoError(t, err)
+ require.Equal(t, 12, got.DMLCount)
+ require.Equal(t, statements, got.DMLStatements)
+}
+
+func parseASTs(t *testing.T, statement string) []base.AST {
+ t.Helper()
+ stmts, err := base.ParseStatements(storepb.Engine_COCKROACHDB, statement)
+ require.NoError(t, err)
+ return base.ExtractASTs(stmts)
+}
+
+// getTableNames returns the changed tables as sorted database.schema.table names.
+func getTableNames(changedResources *model.ChangedResources) []string {
+ var names []string
+ for _, database := range changedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ names = append(names, database.GetName()+"."+schema.GetName()+"."+table.GetName())
+ }
+ }
+ }
+ slices.Sort(names)
+ return names
+}
diff --git a/backend/plugin/parser/cockroachdb/statement_type.go b/backend/plugin/parser/cockroachdb/statement_type.go
new file mode 100644
index 0000000000..4a997696e6
--- /dev/null
+++ b/backend/plugin/parser/cockroachdb/statement_type.go
@@ -0,0 +1,175 @@
+package cockroachdb
+
+import (
+ "slices"
+
+ "github.com/cockroachdb/cockroachdb-parser/pkg/sql/sem/tree"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterGetStatementTypes(storepb.Engine_COCKROACHDB, GetStatementTypes)
+}
+
+// GetStatementTypes returns the types of the statements that have one, in order, each followed by
+// the types of the mutations in its CTEs and statement sources, each type once per statement.
+func GetStatementTypes(asts []base.AST) ([]storepb.StatementType, error) {
+ var statementTypes []storepb.StatementType
+ for _, ast := range asts {
+ crdbAST, ok := ast.(*AST)
+ if !ok {
+ return nil, errors.New("expected CockroachDB AST")
+ }
+ var types []storepb.StatementType
+ add := func(statementType storepb.StatementType) {
+ if statementType != storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED && !slices.Contains(types, statementType) {
+ types = append(types, statementType)
+ }
+ }
+ stmt := crdbAST.Stmt.AST
+ // EXPLAIN ANALYZE executes the statement it explains.
+ if explain, ok := stmt.(*tree.ExplainAnalyze); ok {
+ stmt = explain.Statement
+ }
+ add(getStatementType(stmt))
+ if createTable, ok := stmt.(*tree.CreateTable); ok && createTable.AsSource != nil {
+ stmt = createTable.AsSource
+ }
+ for _, mutationType := range getMutationTypes(stmt) {
+ add(mutationType)
+ }
+ statementTypes = append(statementTypes, types...)
+ }
+ return statementTypes, nil
+}
+
+func getStatementType(stmt tree.Statement) storepb.StatementType {
+ switch n := stmt.(type) {
+ case *tree.CreateDatabase:
+ return storepb.StatementType_CREATE_DATABASE
+ case *tree.CreateTable:
+ return storepb.StatementType_CREATE_TABLE
+ case *tree.CreateView:
+ return storepb.StatementType_CREATE_VIEW
+ case *tree.CreateIndex:
+ return storepb.StatementType_CREATE_INDEX
+ case *tree.CreateSequence:
+ return storepb.StatementType_CREATE_SEQUENCE
+ case *tree.CreateSchema:
+ return storepb.StatementType_CREATE_SCHEMA
+ case *tree.CreateRoutine:
+ // Procedures are functions here, as PostgreSQL classifies CREATE PROCEDURE.
+ return storepb.StatementType_CREATE_FUNCTION
+ case *tree.CreateTrigger:
+ return storepb.StatementType_CREATE_TRIGGER
+ case *tree.CreateExtension:
+ return storepb.StatementType_CREATE_EXTENSION
+ case *tree.CreateType:
+ return storepb.StatementType_CREATE_TYPE
+
+ case *tree.DropDatabase:
+ return storepb.StatementType_DROP_DATABASE
+ case *tree.DropTable:
+ return storepb.StatementType_DROP_TABLE
+ case *tree.DropView:
+ // A materialized view holds data, so dropping one is as risky as dropping a table.
+ if n.IsMaterialized {
+ return storepb.StatementType_DROP_TABLE
+ }
+ return storepb.StatementType_DROP_VIEW
+ case *tree.DropIndex:
+ return storepb.StatementType_DROP_INDEX
+ case *tree.DropSequence:
+ return storepb.StatementType_DROP_SEQUENCE
+ case *tree.DropSchema:
+ return storepb.StatementType_DROP_SCHEMA
+ case *tree.DropType:
+ return storepb.StatementType_DROP_TYPE
+ case *tree.DropTrigger:
+ return storepb.StatementType_DROP_TRIGGER
+ case *tree.DropRoutine:
+ return storepb.StatementType_DROP_FUNCTION
+ case *tree.Truncate:
+ return storepb.StatementType_TRUNCATE
+
+ case *tree.RenameDatabase, *tree.AlterDatabaseOwner, *tree.ReparentDatabase,
+ *tree.AlterDatabaseAddRegion, *tree.AlterDatabaseDropRegion, *tree.AlterDatabasePrimaryRegion,
+ *tree.AlterDatabaseSurvivalGoal, *tree.AlterDatabasePlacement, *tree.AlterDatabaseAddSuperRegion,
+ *tree.AlterDatabaseDropSuperRegion, *tree.AlterDatabaseAlterSuperRegion, *tree.AlterDatabaseSecondaryRegion,
+ *tree.AlterDatabaseDropSecondaryRegion, *tree.AlterDatabaseSetZoneConfigExtension:
+ return storepb.StatementType_ALTER_DATABASE
+ case *tree.SetZoneConfig:
+ switch {
+ case n.Database != "":
+ return storepb.StatementType_ALTER_DATABASE
+ case n.TableOrIndex.Index != "":
+ return storepb.StatementType_ALTER_INDEX
+ case n.TableOrIndex.Table.ObjectName != "":
+ return storepb.StatementType_ALTER_TABLE
+ default:
+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
+ }
+ case *tree.AlterRoleSet:
+ // ALTER DATABASE d SET parses as ALTER ROLE ALL IN DATABASE d SET, which sets the same defaults.
+ if n.AllRoles && n.DatabaseName != "" {
+ return storepb.StatementType_ALTER_DATABASE
+ }
+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
+ case *tree.AlterTable, *tree.AlterTableLocality:
+ return storepb.StatementType_ALTER_TABLE
+ case *tree.AlterTableSetSchema:
+ return getRelationAlterType(n.IsView && !n.IsMaterialized, n.IsSequence)
+ case *tree.AlterTableOwner:
+ return getRelationAlterType(n.IsView && !n.IsMaterialized, n.IsSequence)
+ case *tree.AlterIndex, *tree.AlterIndexVisible:
+ return storepb.StatementType_ALTER_INDEX
+ case *tree.AlterSequence:
+ return storepb.StatementType_ALTER_SEQUENCE
+ case *tree.AlterType:
+ return storepb.StatementType_ALTER_TYPE
+
+ case *tree.RenameTable:
+ switch {
+ case n.IsSequence:
+ return storepb.StatementType_RENAME_SEQUENCE
+ case n.IsView && !n.IsMaterialized:
+ return storepb.StatementType_ALTER_VIEW
+ default:
+ return storepb.StatementType_ALTER_TABLE
+ }
+ case *tree.RenameIndex:
+ return storepb.StatementType_RENAME_INDEX
+ case *tree.AlterSchema:
+ if _, ok := n.Cmd.(*tree.AlterSchemaRename); ok {
+ return storepb.StatementType_RENAME_SCHEMA
+ }
+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
+
+ case *tree.CommentOnColumn, *tree.CommentOnConstraint, *tree.CommentOnDatabase, *tree.CommentOnIndex,
+ *tree.CommentOnSchema, *tree.CommentOnTable, *tree.CommentOnType:
+ return storepb.StatementType_COMMENT
+
+ case *tree.Insert:
+ return storepb.StatementType_INSERT
+ case *tree.Update:
+ return storepb.StatementType_UPDATE
+ case *tree.Delete:
+ return storepb.StatementType_DELETE
+ default:
+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
+ }
+}
+
+func getRelationAlterType(isView bool, isSequence bool) storepb.StatementType {
+ switch {
+ case isSequence:
+ return storepb.StatementType_ALTER_SEQUENCE
+ case isView:
+ return storepb.StatementType_ALTER_VIEW
+ default:
+ return storepb.StatementType_ALTER_TABLE
+ }
+}
diff --git a/backend/plugin/parser/cockroachdb/statement_type_test.go b/backend/plugin/parser/cockroachdb/statement_type_test.go
new file mode 100644
index 0000000000..f3770ee801
--- /dev/null
+++ b/backend/plugin/parser/cockroachdb/statement_type_test.go
@@ -0,0 +1,96 @@
+package cockroachdb
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+func TestGetStatementTypes(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ want []storepb.StatementType
+ }{
+ {statement: `CREATE DATABASE d`, want: []storepb.StatementType{storepb.StatementType_CREATE_DATABASE}},
+ {statement: `CREATE TABLE t (id INT PRIMARY KEY)`, want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE}},
+ {statement: `CREATE TABLE t2 AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE}},
+ {statement: `CREATE VIEW v AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_VIEW}},
+ {statement: `CREATE MATERIALIZED VIEW mv AS SELECT * FROM t`, want: []storepb.StatementType{storepb.StatementType_CREATE_VIEW}},
+ {statement: `CREATE INDEX ON t (c)`, want: []storepb.StatementType{storepb.StatementType_CREATE_INDEX}},
+ {statement: `CREATE SEQUENCE s`, want: []storepb.StatementType{storepb.StatementType_CREATE_SEQUENCE}},
+ {statement: `CREATE SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_CREATE_SCHEMA}},
+ {statement: `CREATE FUNCTION f() RETURNS INT LANGUAGE SQL AS 'SELECT 1'`, want: []storepb.StatementType{storepb.StatementType_CREATE_FUNCTION}},
+ {statement: `CREATE PROCEDURE p() LANGUAGE SQL AS 'SELECT 1'`, want: []storepb.StatementType{storepb.StatementType_CREATE_FUNCTION}},
+ {statement: `CREATE TRIGGER tr BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION f()`, want: []storepb.StatementType{storepb.StatementType_CREATE_TRIGGER}},
+ {statement: `CREATE EXTENSION IF NOT EXISTS pg_trgm`, want: []storepb.StatementType{storepb.StatementType_CREATE_EXTENSION}},
+ {statement: `CREATE TYPE e AS ENUM ('a', 'b')`, want: []storepb.StatementType{storepb.StatementType_CREATE_TYPE}},
+ {statement: `DROP DATABASE d`, want: []storepb.StatementType{storepb.StatementType_DROP_DATABASE}},
+ {statement: `DROP TABLE t`, want: []storepb.StatementType{storepb.StatementType_DROP_TABLE}},
+ {statement: `DROP MATERIALIZED VIEW mv`, want: []storepb.StatementType{storepb.StatementType_DROP_TABLE}},
+ {statement: `DROP VIEW v`, want: []storepb.StatementType{storepb.StatementType_DROP_VIEW}},
+ {statement: `DROP INDEX t@idx`, want: []storepb.StatementType{storepb.StatementType_DROP_INDEX}},
+ {statement: `DROP SEQUENCE s`, want: []storepb.StatementType{storepb.StatementType_DROP_SEQUENCE}},
+ {statement: `DROP SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_DROP_SCHEMA}},
+ {statement: `DROP TYPE e`, want: []storepb.StatementType{storepb.StatementType_DROP_TYPE}},
+ {statement: `DROP TRIGGER tr ON t`, want: []storepb.StatementType{storepb.StatementType_DROP_TRIGGER}},
+ {statement: `DROP FUNCTION f`, want: []storepb.StatementType{storepb.StatementType_DROP_FUNCTION}},
+ {statement: `TRUNCATE t`, want: []storepb.StatementType{storepb.StatementType_TRUNCATE}},
+ {statement: `ALTER DATABASE d RENAME TO d2`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER DATABASE d OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER DATABASE d PRIMARY REGION "us-east1"`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER DATABASE d SURVIVE REGION FAILURE`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER DATABASE d CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER DATABASE d SET timezone = 'UTC'`, want: []storepb.StatementType{storepb.StatementType_ALTER_DATABASE}},
+ {statement: `ALTER ROLE u SET timezone = 'UTC'`},
+ {statement: `ALTER TABLE t CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER INDEX t@idx CONFIGURE ZONE USING num_replicas = 5`, want: []storepb.StatementType{storepb.StatementType_ALTER_INDEX}},
+ {statement: `ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5`},
+ {statement: `ALTER TABLE t ADD COLUMN c2 INT`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER TABLE t RENAME COLUMN c TO c2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER TABLE t SET LOCALITY REGIONAL BY ROW`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER TABLE t SET SCHEMA app`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER VIEW v OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_VIEW}},
+ {statement: `ALTER SEQUENCE s OWNER TO u`, want: []storepb.StatementType{storepb.StatementType_ALTER_SEQUENCE}},
+ {statement: `ALTER INDEX t@idx NOT VISIBLE`, want: []storepb.StatementType{storepb.StatementType_ALTER_INDEX}},
+ {statement: `ALTER SEQUENCE s RESTART WITH 100`, want: []storepb.StatementType{storepb.StatementType_ALTER_SEQUENCE}},
+ {statement: `ALTER TYPE e ADD VALUE 'c'`, want: []storepb.StatementType{storepb.StatementType_ALTER_TYPE}},
+ {statement: `ALTER TABLE t RENAME TO t2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER VIEW v RENAME TO v2`, want: []storepb.StatementType{storepb.StatementType_ALTER_VIEW}},
+ {statement: `ALTER MATERIALIZED VIEW mv RENAME TO mv2`, want: []storepb.StatementType{storepb.StatementType_ALTER_TABLE}},
+ {statement: `ALTER SEQUENCE s RENAME TO s2`, want: []storepb.StatementType{storepb.StatementType_RENAME_SEQUENCE}},
+ {statement: `ALTER INDEX t@idx RENAME TO idx2`, want: []storepb.StatementType{storepb.StatementType_RENAME_INDEX}},
+ {statement: `ALTER SCHEMA app RENAME TO app2`, want: []storepb.StatementType{storepb.StatementType_RENAME_SCHEMA}},
+ {statement: `COMMENT ON TABLE t IS 'comment'`, want: []storepb.StatementType{storepb.StatementType_COMMENT}},
+ {statement: `COMMENT ON COLUMN t.c IS 'comment'`, want: []storepb.StatementType{storepb.StatementType_COMMENT}},
+ {statement: `INSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},
+ {statement: `UPSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},
+ {statement: `UPDATE t SET c = 1`, want: []storepb.StatementType{storepb.StatementType_UPDATE}},
+ {statement: `DELETE FROM t WHERE id = 1`, want: []storepb.StatementType{storepb.StatementType_DELETE}},
+ {
+ statement: `SET application_name = 'x'; CREATE TABLE t (id INT PRIMARY KEY); SELECT 1; INSERT INTO t VALUES (1)`,
+ want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE, storepb.StatementType_INSERT},
+ },
+ {statement: `WITH d AS (DELETE FROM t RETURNING id) SELECT count(*) FROM d`, want: []storepb.StatementType{storepb.StatementType_DELETE}},
+ {statement: `SELECT * FROM [UPDATE t SET c = 1 RETURNING id]`, want: []storepb.StatementType{storepb.StatementType_UPDATE}},
+ {
+ statement: `WITH d AS (DELETE FROM t RETURNING *) INSERT INTO t2 SELECT * FROM d`,
+ want: []storepb.StatementType{storepb.StatementType_INSERT, storepb.StatementType_DELETE},
+ },
+ {
+ statement: `CREATE TABLE t_moved AS SELECT * FROM [DELETE FROM t RETURNING *]`,
+ want: []storepb.StatementType{storepb.StatementType_CREATE_TABLE, storepb.StatementType_DELETE},
+ },
+ {statement: `EXPLAIN ANALYZE DELETE FROM t WHERE id = 1`, want: []storepb.StatementType{storepb.StatementType_DELETE}},
+ {statement: `EXPLAIN ANALYZE (VERBOSE) UPSERT INTO t VALUES (1)`, want: []storepb.StatementType{storepb.StatementType_INSERT}},
+ {statement: `EXPLAIN DELETE FROM t WHERE id = 1`},
+ {statement: `SELECT * FROM t`},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ got, err := GetStatementTypes(parseASTs(t, tc.statement))
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
diff --git a/backend/plugin/parser/mysql/explain_plan.go b/backend/plugin/parser/mysql/explain_plan.go
index d35db6059b..9a3e94f5f5 100644
--- a/backend/plugin/parser/mysql/explain_plan.go
+++ b/backend/plugin/parser/mysql/explain_plan.go
@@ -3,105 +3,578 @@ package mysql
import (
"encoding/json"
"strconv"
+ "strings"
+ "unicode"
+
+ "github.com/bytebase/omni/mysql/ast"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
)
-// GetEstimatedAffectedRowsFromExplainJSON estimates the number of rows affected by an
-// UPDATE/DELETE statement from `EXPLAIN FORMAT=JSON` output.
-//
-// MySQL and MariaDB flag the plan node of each DML target table with "update"/"delete"
-// (boolean true on MySQL, integer 1 on MariaDB). Unlike the first "rows" value of the
-// tabular EXPLAIN output — which may be the scan estimate of a driving table unrelated
-// to the target — the target node carries the optimizer's cardinality estimate for the
-// rows actually reaching the modify step. Estimates of multiple target nodes
-// (multi-table DELETE/UPDATE) are summed.
-//
-// The boolean result reports whether any flagged target node with a row estimate was
-// found; callers should fall back to the tabular-EXPLAIN heuristic when it is false.
-func GetEstimatedAffectedRowsFromExplainJSON(plan string) (int64, bool) {
- var root any
+// AffectedRowsQuery returns the statement to EXPLAIN for the rows stmt, parsed from statement,
+// modifies. MySQL and MariaDB estimate the rows a single-table UPDATE or DELETE scans rather than
+// the rows its WHERE clause keeps, so such a statement is explained as a SELECT of those rows.
+func AffectedRowsQuery(stmt ast.Node, statement string) string {
+ var loc ast.Loc
+ var table *ast.TableRef
+ // clausesStart is where the WHERE, ORDER BY, and LIMIT clauses begin.
+ var clausesStart int
+ switch s := stmt.(type) {
+ case *ast.UpdateStmt:
+ table = singleTableRef(s.Tables)
+ if table == nil || len(s.SetList) == 0 {
+ return statement
+ }
+ loc, clausesStart = s.Loc, s.SetList[len(s.SetList)-1].Loc.End
+ case *ast.DeleteStmt:
+ table = singleTableRef(s.Tables)
+ // DELETE writes an alias before PARTITION, which a SELECT writes after it.
+ if table == nil || len(s.Using) > 0 || (table.Alias != "" && len(table.Partitions) > 0) {
+ return statement
+ }
+ loc, clausesStart = s.Loc, table.Loc.End
+ default:
+ return statement
+ }
+ if loc.Start < 0 || loc.Start > table.Loc.Start || table.Loc.End > clausesStart || clausesStart > loc.End || loc.End > len(statement) {
+ return statement
+ }
+ // omni positions the text after an executable comment as if the comment's markers were removed, so
+ // no SELECT is cut from a statement with one. The SELECT would also leave out an optimizer hint
+ // before the table, which can change the plan.
+ if strings.Contains(statement, "/*!") || strings.Contains(statement, "/*M!") || strings.Contains(statement[loc.Start:table.Loc.Start], "/*+") {
+ return statement
+ }
+ clauses := strings.TrimSpace(statement[clausesStart:loc.End])
+ if rest := trimLeadingComments(clauses); rest != "" && !startsWithKeyword(rest, "WHERE", "ORDER", "LIMIT") {
+ return statement
+ }
+ // The text before the statement holds its WITH clause.
+ query := statement[:loc.Start] + "SELECT 1 FROM " + statement[table.Loc.Start:table.Loc.End]
+ if clauses != "" {
+ query += " " + clauses
+ }
+ return query
+}
+
+func singleTableRef(tables []ast.TableExpr) *ast.TableRef {
+ if len(tables) != 1 {
+ return nil
+ }
+ if table, ok := tables[0].(*ast.TableRef); ok {
+ return table
+ }
+ return nil
+}
+
+// trimLeadingComments removes the whitespace and ordinary comments at the start of text, where -- opens
+// a comment only before whitespace or a control character. Executable comments and optimizer hints
+// stay, because they can change the statement.
+func trimLeadingComments(text string) string {
+ for {
+ text = strings.TrimLeftFunc(text, unicode.IsSpace)
+ switch {
+ case strings.HasPrefix(text, "/*!") || strings.HasPrefix(text, "/*M!") || strings.HasPrefix(text, "/*+"):
+ return text
+ case strings.HasPrefix(text, "/*"):
+ end := strings.Index(text[2:], "*/")
+ if end < 0 {
+ return text
+ }
+ text = text[2+end+2:]
+ case (strings.HasPrefix(text, "--") && (len(text) == 2 || text[2] <= ' ')) || strings.HasPrefix(text, "#"):
+ end := strings.IndexByte(text, '\n')
+ if end < 0 {
+ return ""
+ }
+ text = text[end+1:]
+ default:
+ return text
+ }
+ }
+}
+
+func startsWithKeyword(text string, keywords ...string) bool {
+ word := text
+ if i := strings.IndexFunc(text, func(r rune) bool { return !unicode.IsLetter(r) }); i >= 0 {
+ word = text[:i]
+ }
+ for _, keyword := range keywords {
+ if strings.EqualFold(word, keyword) {
+ return true
+ }
+ }
+ return false
+}
+
+// EstimateAffectedRowsFromExplainJSON returns the planner's estimate of the rows an INSERT,
+// REPLACE, UPDATE, or DELETE statement modifies, capped by the statement's LIMIT. The plan is
+// MySQL `EXPLAIN FORMAT=JSON` output in explain_json_format_version 1, or MariaDB's, for the
+// statement's AffectedRowsQuery.
+func EstimateAffectedRowsFromExplainJSON(stmt ast.Node, plan string) (int64, error) {
+ var root map[string]any
if err := json.Unmarshal([]byte(plan), &root); err != nil {
- return 0, false
+ return 0, errors.Wrap(err, "failed to parse EXPLAIN FORMAT=JSON output")
}
- var total float64
+ block, ok := root["query_block"].(map[string]any)
+ if !ok {
+ return 0, errors.New("unsupported EXPLAIN FORMAT=JSON output: no query_block")
+ }
+
+ var rows float64
+ var err error
+ switch s := stmt.(type) {
+ case *ast.InsertStmt:
+ // MySQL prints no source plan for a SELECT that reads no table, such as
+ // SELECT ... FROM DUAL WHERE NOT EXISTS (...).
+ if tableFreeRows, ok := tableFreeSelectRows(s.Select); ok {
+ rows = float64(tableFreeRows)
+ } else {
+ rows, err = queryBlockRows(block)
+ }
+ case *ast.UpdateStmt:
+ rows, err = targetRows(block, updateTargets(s))
+ case *ast.DeleteStmt:
+ rows, err = targetRows(block, deleteTargets(s))
+ default:
+ return 0, errors.Errorf("unsupported statement type %T", stmt)
+ }
+ if err != nil {
+ return 0, err
+ }
+ return CapAffectedRowsByLimit(stmt, rows), nil
+}
+
+// CapAffectedRowsByLimit rounds an estimate of the rows an INSERT, REPLACE, UPDATE, or DELETE
+// statement modifies and caps it by the statement's LIMIT.
+func CapAffectedRowsByLimit(stmt ast.Node, rows float64) int64 {
+ var limit *ast.Limit
+ switch s := stmt.(type) {
+ case *ast.InsertStmt:
+ if s.Select != nil {
+ limit = s.Select.Limit
+ } else if s.TableSource != nil {
+ limit = s.TableSource.Limit
+ }
+ case *ast.UpdateStmt:
+ limit = s.Limit
+ case *ast.DeleteStmt:
+ limit = s.Limit
+ default:
+ }
+ count := common.RoundRows(rows)
+ if limitRows, ok := limitCount(limit); ok && limitRows < count {
+ return limitRows
+ }
+ return count
+}
+
+// EstimateAffectedRowsFromOceanBaseExplainJSON returns the EST.ROWS estimate from OceanBase
+// `EXPLAIN FORMAT=JSON` output: the root operator's, or the largest child operator's when the
+// root has no operator.
+func EstimateAffectedRowsFromOceanBaseExplainJSON(plan string) (int64, error) {
+ var root map[string]any
+ if err := json.Unmarshal([]byte(plan), &root); err != nil {
+ return 0, errors.Wrap(err, "failed to parse EXPLAIN FORMAT=JSON output")
+ }
+ if operator, ok := root["OPERATOR"].(string); ok && operator != "" {
+ rows, ok := planNumber(root["EST.ROWS"])
+ if !ok {
+ return 0, errors.Errorf("operator %q has no EST.ROWS", operator)
+ }
+ return common.RoundRows(rows), nil
+ }
+ var maxRows float64
found := false
- var walk func(node any)
- walk = func(node any) {
+ for key, value := range root {
+ child, ok := value.(map[string]any)
+ if !ok || !strings.HasPrefix(key, "CHILD_") {
+ continue
+ }
+ operator, ok := child["OPERATOR"].(string)
+ if !ok || operator == "" {
+ continue
+ }
+ rows, ok := planNumber(child["EST.ROWS"])
+ if !ok {
+ return 0, errors.Errorf("operator %q has no EST.ROWS", operator)
+ }
+ if !found || rows > maxRows {
+ maxRows = rows
+ found = true
+ }
+ }
+ if !found {
+ return 0, errors.New("the plan has no operator")
+ }
+ return common.RoundRows(maxRows), nil
+}
+
+// queryBlockRows returns the planner's estimate of the rows a query block produces. A UNION sums its
+// branches, an INTERSECT takes the fewest, and an EXCEPT, or a parenthesized query with its own
+// ORDER BY or LIMIT, takes its first.
+func queryBlockRows(block map[string]any) (float64, error) {
+ for _, operation := range []string{"union_result", "intersect_result", "except_result", "unary_result"} {
+ result, ok := block[operation].(map[string]any)
+ if !ok {
+ continue
+ }
+ branches, ok := result["query_specifications"].([]any)
+ if !ok || len(branches) == 0 {
+ return 0, errors.Errorf("%s has no query_specifications", operation)
+ }
+ var total float64
+ for i, branch := range branches {
+ specification, ok := branch.(map[string]any)
+ if !ok {
+ return 0, errors.New("unexpected query_specifications element")
+ }
+ // MySQL nests a set operation as a branch without a query_block of its own.
+ branchBlock, ok := specification["query_block"].(map[string]any)
+ if !ok {
+ branchBlock = specification
+ }
+ rows, err := queryBlockRows(branchBlock)
+ if err != nil {
+ return 0, err
+ }
+ switch {
+ case i == 0 || operation == "union_result":
+ total += rows
+ case operation == "intersect_result":
+ total = min(total, rows)
+ default:
+ }
+ }
+ return total, nil
+ }
+ _, cumulative, err := planJoin(block)
+ if err != nil {
+ return 0, err
+ }
+ return cumulative[len(cumulative)-1], nil
+}
+
+// dmlTargets holds the names a plan uses for the tables an UPDATE or DELETE modifies.
+type dmlTargets struct {
+ names []string
+ // unqualified counts the multi-table UPDATE assignments to a column without a table qualifier.
+ unqualified int
+}
+
+// DMLTargetCount returns how many tables stmt can change for each row its join produces: the targets
+// of a multi-table UPDATE or DELETE, counting each unqualified assignment as another joined table, up
+// to joinedTables, and one for any other statement.
+func DMLTargetCount(stmt ast.Node, joinedTables int) int {
+ var targets dmlTargets
+ switch s := stmt.(type) {
+ case *ast.UpdateStmt:
+ targets = updateTargets(s)
+ case *ast.DeleteStmt:
+ targets = deleteTargets(s)
+ default:
+ return 1
+ }
+ return max(min(len(targets.names)+targets.unqualified, joinedTables), 1)
+}
+
+func updateTargets(stmt *ast.UpdateStmt) dmlTargets {
+ if len(stmt.Tables) == 1 {
+ if table, ok := stmt.Tables[0].(*ast.TableRef); ok {
+ return dmlTargets{names: []string{tableRefPlanName(table)}}
+ }
+ }
+ var targets dmlTargets
+ for _, assignment := range stmt.SetList {
+ if assignment.Column == nil || assignment.Column.Table == "" {
+ targets.unqualified++
+ continue
+ }
+ if !containsFold(targets.names, assignment.Column.Table) {
+ targets.names = append(targets.names, assignment.Column.Table)
+ }
+ }
+ return targets
+}
+
+func deleteTargets(stmt *ast.DeleteStmt) dmlTargets {
+ var targets dmlTargets
+ for _, expr := range stmt.Tables {
+ table, ok := expr.(*ast.TableRef)
+ if !ok {
+ continue
+ }
+ // The multi-table forms list names declared in FROM or USING, so only the single-table
+ // form can declare an alias here.
+ name := table.Name
+ if len(stmt.Using) == 0 {
+ name = tableRefPlanName(table)
+ }
+ targets.names = append(targets.names, name)
+ }
+ return targets
+}
+
+func targetRows(block map[string]any, targets dmlTargets) (float64, error) {
+ tables, cumulative, err := planJoin(block)
+ if err != nil {
+ return 0, err
+ }
+ final := cumulative[len(cumulative)-1]
+ // A plan that produces no rows, such as "Impossible WHERE", prints no target.
+ if final == 0 {
+ return 0, nil
+ }
+
+ // Each target counts min(its cumulative estimate, the join's final estimate): the rows
+ // reaching the modification, bounded by the rows read from the target table.
+ var total float64
+ flagged := false
+ for i, table := range tables {
+ if planFlag(table["update"]) || planFlag(table["delete"]) {
+ total += min(cumulative[i], final)
+ flagged = true
+ }
+ }
+ if flagged {
+ return total, nil
+ }
+
+ // MariaDB flags only single-table targets, and the SELECT a single-table UPDATE or DELETE is
+ // explained as flags none, so find the targets by name. A name that no plan table has, such as a
+ // view's, or that several have, as when a subquery reads another table of that name, counts the
+ // final estimate.
+ if len(targets.names) == 0 && targets.unqualified == 0 {
+ return 0, errors.New("the plan has no UPDATE or DELETE target")
+ }
+ for _, name := range targets.names {
+ index, matches := -1, 0
+ for i, table := range tables {
+ if tableName, ok := table["table_name"].(string); ok && strings.EqualFold(tableName, name) {
+ index = i
+ matches++
+ }
+ }
+ if matches == 1 {
+ total += min(cumulative[index], final)
+ } else {
+ total += final
+ }
+ }
+ // An unqualified column may belong to any joined table, so each counts the final estimate, up to
+ // one per joined table.
+ total += final * float64(min(targets.unqualified, len(tables)))
+ return total, nil
+}
+
+// planWrappers are plan members that wrap a query body without changing the tables it joins.
+var planWrappers = []string{
+ // MySQL.
+ "ordering_operation", "grouping_operation", "duplicates_removal", "windowing", "buffer_result",
+ // MariaDB.
+ "block-nl-join", "read_sorted_file", "filesort", "temporary_table", "window_functions_computation", "buffer",
+}
+
+// planJoin returns the tables a query block joins, in join order, with the planner's estimate of
+// the rows produced after joining each one. It does not descend into subqueries or derived tables.
+func planJoin(block map[string]any) ([]map[string]any, []float64, error) {
+ var tables []map[string]any
+ var collect func(node any) error
+ collect = func(node any) error {
switch n := node.(type) {
+ case []any:
+ for _, element := range n {
+ if err := collect(element); err != nil {
+ return err
+ }
+ }
+ return nil
case map[string]any:
- if isDMLTargetNode(n) {
- if estimate, ok := targetNodeRowEstimate(n); ok {
- total += estimate
- found = true
+ // A MySQL INSERT plan puts the target table beside insert_from, which holds the source.
+ for _, key := range []string{"insert_from", "nested_loop"} {
+ if body, ok := n[key]; ok {
+ return collect(body)
}
}
- for _, v := range n {
- walk(v)
+ if table, ok := n["table"].(map[string]any); ok {
+ tables = append(tables, table)
+ return nil
}
- case []any:
- for _, v := range n {
- walk(v)
+ for _, key := range planWrappers {
+ if body, ok := n[key]; ok {
+ return collect(body)
+ }
+ }
+ if _, ok := n["message"].(string); ok {
+ tables = append(tables, n)
+ return nil
}
+ return errors.New("unrecognized EXPLAIN FORMAT=JSON plan node")
default:
+ return errors.Errorf("unexpected EXPLAIN FORMAT=JSON plan value %v", node)
}
}
- walk(root)
- if !found {
- return 0, false
+ if err := collect(block); err != nil {
+ return nil, nil, err
+ }
+ if len(tables) == 0 {
+ return nil, nil, errors.New("the plan has no tables")
}
- return int64(total), true
-}
-func isDMLTargetNode(node map[string]any) bool {
- for _, key := range []string{"update", "delete"} {
- switch v := node[key].(type) {
- case bool:
- if v {
- return true
+ cumulative := make([]float64, len(tables))
+ previous, fanout := 1.0, 1.0
+ for i, table := range tables {
+ filtered, ok := planNumber(table["filtered"])
+ if !ok {
+ filtered = 100
+ }
+ produced, hasProduced := planNumber(table["rows_produced_per_join"])
+ scanned, hasScanned := planNumber(table["rows_examined_per_scan"])
+ if !hasScanned {
+ scanned, hasScanned = planNumber(table["rows"])
+ }
+ materialized, isMaterialized := table["materialized_from_subquery"].(map[string]any)
+ message, hasMessage := table["message"].(string)
+ switch {
+ case hasProduced:
+ cumulative[i] = produced * fanout
+ case hasScanned:
+ // MariaDB prints loops; otherwise every row produced so far drives one scan.
+ loops, ok := planNumber(table["loops"])
+ if !ok {
+ loops = previous
+ }
+ cumulative[i] = loops * scanned * filtered / 100
+ case isMaterialized:
+ // A MySQL semi-join materialization scan has no row estimate of its own, and the
+ // per-join estimates of the tables after it count rows per materialized row.
+ subquery, ok := materialized["query_block"].(map[string]any)
+ if !ok {
+ return nil, nil, errors.New("materialized_from_subquery has no query_block")
+ }
+ rows, err := queryBlockRows(subquery)
+ if err != nil {
+ return nil, nil, err
}
- case float64:
- if v != 0 {
- return true
+ fanout *= rows
+ cumulative[i] = previous * rows
+ case hasMessage:
+ rows, ok := planMessageRows(message)
+ if !ok {
+ return nil, nil, errors.Errorf("the plan has no row estimate: %s", message)
}
+ cumulative[i] = previous * rows
default:
+ if tableName, ok := table["table_name"].(string); ok {
+ return nil, nil, errors.Errorf("table %q in the plan has no row estimate", tableName)
+ }
+ return nil, nil, errors.New("the plan has a table without a row estimate")
}
+ previous = cumulative[i]
}
- return false
+ return tables, cumulative, nil
}
-// targetNodeRowEstimate prefers "rows_produced_per_join", the cumulative cardinality
-// after all join and filter conditions up to this node. When absent (single-table
-// plans, MariaDB), it scales the per-scan row estimate by the node's "filtered"
-// percentage.
-func targetNodeRowEstimate(node map[string]any) (float64, bool) {
- if v, ok := toFloat(node["rows_produced_per_join"]); ok {
- return v, true
- }
- base, ok := toFloat(node["rows_examined_per_scan"])
- if !ok {
- // MariaDB names the per-scan estimate "rows".
- base, ok = toFloat(node["rows"])
- }
- if !ok {
+// planMessageRows returns the rows produced where the plan prints an optimizer message in place
+// of a table.
+func planMessageRows(message string) (float64, bool) {
+ switch message {
+ case "Impossible WHERE", "Impossible WHERE noticed after reading const tables",
+ "Impossible HAVING", "Impossible HAVING noticed after reading const tables",
+ "no matching row in const table", "No matching rows after partition pruning", "Zero limit":
+ return 0, true
+ case "No tables used", "Select tables optimized away":
+ return 1, true
+ default:
return 0, false
}
- if filtered, ok := toFloat(node["filtered"]); ok {
- base = base * filtered / 100
+}
+
+// planFlag reads an update/delete marker, which MySQL prints as true and MariaDB as 1.
+func planFlag(value any) bool {
+ switch v := value.(type) {
+ case bool:
+ return v
+ case float64:
+ return v != 0
+ default:
+ return false
}
- return base, true
}
-func toFloat(v any) (float64, bool) {
- switch value := v.(type) {
+// planNumber reads a plan number, which MySQL sometimes prints as a string such as "10.00".
+func planNumber(value any) (float64, bool) {
+ switch v := value.(type) {
case float64:
- return value, true
+ return v, true
case string:
- // MySQL renders "filtered" as a string such as "10.00".
- f, err := strconv.ParseFloat(value, 64)
- if err != nil {
+ f, err := strconv.ParseFloat(v, 64)
+ return f, err == nil
+ default:
+ return 0, false
+ }
+}
+
+// tableFreeSelectRows returns the most rows a SELECT produces when none of its branches reads a
+// table: one for each branch, combined as queryBlockRows combines set operations.
+func tableFreeSelectRows(sel *ast.SelectStmt) (int, bool) {
+ if sel == nil {
+ return 0, false
+ }
+ if sel.ParenSource != nil {
+ return tableFreeSelectRows(sel.ParenSource)
+ }
+ if sel.SetOp != ast.SetOpNone {
+ left, leftOK := tableFreeSelectRows(sel.Left)
+ right, rightOK := tableFreeSelectRows(sel.Right)
+ if !leftOK || !rightOK {
return 0, false
}
- return f, true
- default:
+ switch sel.SetOp {
+ case ast.SetOpIntersect:
+ return min(left, right), true
+ case ast.SetOpExcept:
+ return left, true
+ default:
+ return left + right, true
+ }
+ }
+ if sel.TableSource != nil || sel.ValuesSource != nil {
+ return 0, false
+ }
+ for _, from := range sel.From {
+ table, ok := from.(*ast.TableRef)
+ if !ok || table.Schema != "" || !strings.EqualFold(table.Name, "dual") {
+ return 0, false
+ }
+ }
+ return 1, true
+}
+
+func tableRefPlanName(table *ast.TableRef) string {
+ if table.Alias != "" {
+ return table.Alias
+ }
+ return table.Name
+}
+
+func containsFold(names []string, name string) bool {
+ for _, n := range names {
+ if strings.EqualFold(n, name) {
+ return true
+ }
+ }
+ return false
+}
+
+func limitCount(limit *ast.Limit) (int64, bool) {
+ if limit == nil || limit.Count == nil {
+ return 0, false
+ }
+ lit, ok := limit.Count.(*ast.IntLit)
+ if !ok || lit.Value < 0 {
return 0, false
}
+ return lit.Value, true
}
diff --git a/backend/plugin/parser/mysql/explain_plan_test.go b/backend/plugin/parser/mysql/explain_plan_test.go
index 3447b4bffa..191193b880 100644
--- a/backend/plugin/parser/mysql/explain_plan_test.go
+++ b/backend/plugin/parser/mysql/explain_plan_test.go
@@ -1,93 +1,584 @@
package mysql
import (
+ "math"
"os"
"path/filepath"
"testing"
+ "github.com/bytebase/omni/mysql/ast"
"github.com/stretchr/testify/require"
)
-// The fixtures are real `EXPLAIN FORMAT=JSON` outputs captured from MySQL 8.0 and
-// MariaDB 11 against a 400k-row target_table joined with a 100k-row related_table.
-func TestGetEstimatedAffectedRowsFromExplainJSON(t *testing.T) {
+// The fixtures are real `EXPLAIN FORMAT=JSON` outputs. Plans over target_table (400k rows) and
+// related_table (100k rows) come from MySQL 8.0 and MariaDB 11. The others come from MySQL 8.0.33
+// (unprefixed), MySQL 5.7, and MariaDB 11.8 against t (1000 rows), s (100 rows, 10 flagged), big
+// (10000 rows, 100 per s row), small (3 rows), and the empty copies t2 and big2.
+func TestAffectedRowsQuery(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want string
+ }{
+ {
+ name: "update with filter, order, and limit",
+ statement: "UPDATE t SET v = CONCAT(v, 'y'), c = c + 1 WHERE c < 10 ORDER BY id LIMIT 3;",
+ want: "SELECT 1 FROM t WHERE c < 10 ORDER BY id LIMIT 3",
+ },
+ {
+ name: "update of an aliased table with an index hint",
+ statement: "UPDATE t AS tt FORCE INDEX (idx_c) SET tt.v = 'y' WHERE tt.c < 10",
+ want: "SELECT 1 FROM t AS tt FORCE INDEX (idx_c) WHERE tt.c < 10",
+ },
+ {
+ name: "update of every row",
+ statement: "UPDATE t SET d = CASE WHEN c > 1 THEN 1 ELSE 2 END;",
+ want: "SELECT 1 FROM t",
+ },
+ {
+ name: "update with a comment before WHERE",
+ statement: "UPDATE t SET v = (SELECT MAX(v) FROM s) /* latest */ WHERE id = 1",
+ want: "SELECT 1 FROM t WHERE id = 1",
+ },
+ {
+ name: "delete with a comment before WHERE",
+ statement: "DELETE FROM t -- ticket 42\nWHERE id = 1;",
+ want: "SELECT 1 FROM t -- ticket 42\nWHERE id = 1",
+ },
+ {
+ name: "update with a block comment before WHERE",
+ statement: "UPDATE t SET c = 1 /* ticket 42 */ WHERE id = 1;",
+ want: "SELECT 1 FROM t WHERE id = 1",
+ },
+ {
+ name: "delete with an executable comment",
+ statement: "DELETE FROM t /*!80000 WHERE id = 1 */;",
+ want: "DELETE FROM t /*!80000 WHERE id = 1 */;",
+ },
+ {
+ name: "update with an optimizer hint",
+ statement: "UPDATE /*+ SET_VAR(optimizer_switch='condition_fanout_filter=off') */ t SET c = 1 WHERE id > 5;",
+ want: "UPDATE /*+ SET_VAR(optimizer_switch='condition_fanout_filter=off') */ t SET c = 1 WHERE id > 5;",
+ },
+ {
+ name: "delete with an optimizer hint",
+ statement: "DELETE /*+ NO_RANGE_OPTIMIZATION(t) */ FROM t WHERE id > 5;",
+ want: "DELETE /*+ NO_RANGE_OPTIMIZATION(t) */ FROM t WHERE id > 5;",
+ },
+ {
+ // omni positions the text after an executable comment as if its markers were removed.
+ name: "delete with an executable comment in WHERE",
+ statement: "DELETE FROM big WHERE /*!50700 v = 0 AND */ s_id < 10000000000;",
+ want: "DELETE FROM big WHERE /*!50700 v = 0 AND */ s_id < 10000000000;",
+ },
+ {
+ name: "update with a comment before the table",
+ statement: "UPDATE /* ticket 42 */ t SET c = 1 WHERE id > 5;",
+ want: "SELECT 1 FROM t WHERE id > 5",
+ },
+ {
+ name: "update with a CTE",
+ statement: "WITH x AS (SELECT id FROM s WHERE flag = 1) UPDATE t SET v = 'y' WHERE grp IN (SELECT id FROM x);",
+ want: "WITH x AS (SELECT id FROM s WHERE flag = 1) SELECT 1 FROM t WHERE grp IN (SELECT id FROM x)",
+ },
+ {
+ name: "delete from a partition",
+ statement: "DELETE LOW_PRIORITY QUICK IGNORE FROM db1.t PARTITION (p0) WHERE c = 1 ORDER BY id DESC LIMIT 5;",
+ want: "SELECT 1 FROM db1.t PARTITION (p0) WHERE c = 1 ORDER BY id DESC LIMIT 5",
+ },
+ {
+ name: "delete from an aliased table",
+ statement: "DELETE FROM t AS a WHERE a.c = 1;",
+ want: "SELECT 1 FROM t AS a WHERE a.c = 1",
+ },
+ {
+ name: "delete of every row",
+ statement: "DELETE FROM big;",
+ want: "SELECT 1 FROM big",
+ },
+ {
+ name: "delete from a partition of an aliased table",
+ statement: "DELETE FROM t AS a PARTITION (p0) WHERE a.c = 1;",
+ want: "DELETE FROM t AS a PARTITION (p0) WHERE a.c = 1;",
+ },
+ {
+ name: "multi-table update",
+ statement: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1 WHERE s.flag = 1;",
+ want: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1 WHERE s.flag = 1;",
+ },
+ {
+ name: "update of a table list",
+ statement: "UPDATE t, s SET t.v = 'a' WHERE t.id = s.id;",
+ want: "UPDATE t, s SET t.v = 'a' WHERE t.id = s.id;",
+ },
+ {
+ name: "multi-table delete",
+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ want: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ },
+ {
+ name: "delete with USING",
+ statement: "DELETE FROM big USING big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ want: "DELETE FROM big USING big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ },
+ {
+ name: "insert",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ want: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ require.Equal(t, tc.want, AffectedRowsQuery(parseSingleStatement(t, tc.statement), tc.statement))
+ })
+ }
+}
+
+func TestEstimateAffectedRowsFromExplainJSON(t *testing.T) {
for _, tc := range []struct {
fixture string
- wantRows int64
- wantFound bool
+ statement string
+ want int64
}{
{
- // UPDATE ... WHERE ... EXISTS rewritten to a semijoin; the target table is
- // the last nested_loop node. The first tabular EXPLAIN row would report the
- // 100232-row driving-table scan (BYT-9858).
+ // The UPDATE target is the last node of a duplicate-weedout semijoin (BYT-9858).
fixture: "semijoin_update.json",
- wantRows: 1144,
- wantFound: true,
+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",
+ want: 1144,
},
{
- // Single-table DELETE with a range condition: no rows_produced_per_join,
- // estimate is rows_examined_per_scan scaled by filtered.
+ // Single-table plans have no rows_produced_per_join; the scan estimate is scaled by filtered.
fixture: "single_delete.json",
- wantRows: 200,
- wantFound: true,
+ statement: "DELETE FROM target_table WHERE join_key < 50;",
+ want: 200,
},
{
- // UPDATE ... ORDER BY ... LIMIT wraps the target node in ordering_operation.
fixture: "order_limit_update.json",
- wantRows: 200,
- wantFound: true,
+ statement: "UPDATE target_table SET target_flag = 1 WHERE join_key < 50 ORDER BY id LIMIT 500;",
+ want: 200,
},
{
- // NOT EXISTS antijoin: the target table is the first nested_loop node.
fixture: "not_exists_update.json",
- wantRows: 39964,
- wantFound: true,
+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND NOT EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",
+ want: 39964,
},
{
- // IN (SELECT ...) materialization: the subquery plan lives in
- // attached_subqueries and must not contribute to the estimate.
+ // The subquery plan in attached_subqueries does not contribute.
fixture: "in_materialized_update.json",
- wantRows: 399648,
- wantFound: true,
+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.join_key IN (SELECT t.join_key FROM related_table t WHERE t.filter_column_1 = 'VALUE_A' GROUP BY t.join_key HAVING COUNT(*) > 0);",
+ want: 399648,
},
{
- // DELETE o, t FROM ... flags both target nodes; estimates are summed.
+ // Both targets are flagged: min(10023, 38163) + min(38163, 38163).
fixture: "multi_target_delete.json",
- wantRows: 48186,
- wantFound: true,
+ statement: "DELETE o, t FROM target_table o JOIN related_table t ON t.join_key = o.join_key WHERE t.filter_column_1 = 'VALUE_A';",
+ want: 48186,
},
{
fixture: "delete_limit.json",
- wantRows: 10,
- wantFound: true,
+ statement: "DELETE FROM target_table LIMIT 10;",
+ want: 10,
},
{
- // MariaDB marks the target with "update": 1 and names the estimate "rows".
fixture: "mariadb_update.json",
- wantRows: 10000,
- wantFound: true,
+ statement: "UPDATE target_table o SET o.target_flag = 1 WHERE o.target_flag IS NULL AND EXISTS (SELECT 1 FROM related_table t WHERE t.join_key = o.join_key AND t.filter_column_1 = 'VALUE_A');",
+ want: 10000,
+ },
+ {
+ // The plan scans the flagged target first; the later join filter bounds it: min(11330, 1133).
+ fixture: "update_target_first_straight_join.json",
+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",
+ want: 1133,
+ },
+ {
+ // The derived table's fanout does not raise the target past its own estimate: min(1000, 10000).
+ fixture: "update_derived_join.json",
+ statement: "UPDATE t JOIN (SELECT grp, COUNT(*) cnt FROM t GROUP BY grp) g ON g.grp = t.grp SET t.v = 'g' WHERE g.cnt > 50;",
+ want: 1000,
+ },
+ {
+ // The materialization scan prints no rows, and the target's 113 is per materialized row: 113 * 10.
+ fixture: "delete_cte_semijoin_materialization_scan.json",
+ statement: "WITH x AS (SELECT id FROM s WHERE flag = 1) DELETE FROM big WHERE s_id IN (SELECT id FROM x);",
+ want: 1130,
+ },
+ {
+ // The plans of the SELECT that AffectedRowsQuery explains for a single-table change.
+ fixture: "select_for_update_filter.json",
+ statement: "UPDATE t SET v = CONCAT(v, 'y') WHERE d = 3;",
+ want: 100,
+ },
+ {
+ fixture: "mariadb_select_for_delete_range_filter.json",
+ statement: "DELETE FROM t WHERE c < 10 AND d = 3;",
+ want: 14,
+ },
+ {
+ fixture: "mariadb_select_for_delete_all_rows.json",
+ statement: "DELETE FROM big;",
+ want: 10000,
},
{
- // SELECT plans have no update/delete node; callers must fall back.
+ // A target the plan does not name, such as a view, counts the final estimate.
fixture: "select_only.json",
- wantRows: 0,
- wantFound: false,
+ statement: "UPDATE other_table SET target_flag = 1;",
+ want: 200,
+ },
+ {
+ // UNION ALL of 3 joined rows and the first EXCEPT branch's 500.
+ fixture: "insert_select_union_except.json",
+ statement: "INSERT INTO t2 SELECT t.* FROM small JOIN t USING (id) UNION ALL (SELECT * FROM t WHERE c < 50 EXCEPT SELECT * FROM t WHERE d = 1);",
+ want: 503,
+ },
+ {
+ // UNION ALL of 3 joined rows and the smaller INTERSECT branch's 100.
+ fixture: "insert_select_union_intersect.json",
+ statement: "INSERT INTO t2 SELECT t.* FROM small JOIN t USING (id) UNION ALL (SELECT * FROM t WHERE c < 50 INTERSECT SELECT * FROM t WHERE d = 1);",
+ want: 103,
+ },
+ {
+ fixture: "insert_select_parenthesized_limit.json",
+ statement: "INSERT INTO t2 (SELECT * FROM t ORDER BY c LIMIT 500) ORDER BY id LIMIT 400;",
+ want: 400,
+ },
+ {
+ fixture: "update_impossible_where.json",
+ statement: "UPDATE t SET v = 'y' WHERE 1 = 0;",
+ want: 0,
+ },
+ {
+ // MySQL 5.7 prints the row estimate beside the "Deleting all rows" message.
+ fixture: "mysql57_delete_all_rows.json",
+ statement: "DELETE FROM big;",
+ want: 10195,
+ },
+ {
+ fixture: "insert_select_filter.json",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ want: 100,
+ },
+ {
+ fixture: "insert_select_join.json",
+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",
+ want: 1133,
+ },
+ {
+ // Only the first branch sits under insert_from: 100 + 899.
+ fixture: "insert_select_union_all.json",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE c < 10 UNION ALL SELECT * FROM t WHERE c >= 10;",
+ want: 999,
+ },
+ {
+ fixture: "replace_select_join.json",
+ statement: "REPLACE INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",
+ want: 1133,
+ },
+ {
+ fixture: "insert_table.json",
+ statement: "INSERT INTO t2 TABLE t;",
+ want: 1000,
+ },
+ {
+ fixture: "insert_select_join.json",
+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1 LIMIT 30;",
+ want: 30,
+ },
+ {
+ // The lookup into the materialized subquery has no per-join estimate.
+ fixture: "insert_select_semijoin_materialization_lookup.json",
+ statement: "INSERT INTO t2 WITH x AS (SELECT id FROM s WHERE flag = 1) SELECT * FROM t WHERE grp IN (SELECT id FROM x);",
+ want: 1000,
+ },
+ {
+ fixture: "insert_select_buffer_result.json",
+ statement: "INSERT INTO t2 SELECT t.* FROM t LEFT JOIN t2 ON t2.id = t.id WHERE t2.id IS NULL;",
+ want: 1000,
+ },
+ {
+ fixture: "insert_select_impossible_where.json",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE 1 = 0;",
+ want: 0,
+ },
+ {
+ // MariaDB marks no multi-table target, so the target is found by name.
+ fixture: "mariadb_delete_join.json",
+ statement: "DELETE big FROM big JOIN s ON big.s_id = s.id WHERE s.flag = 1;",
+ want: 1000,
+ },
+ {
+ fixture: "mariadb_delete_semijoin.json",
+ statement: "DELETE FROM big WHERE s_id IN (SELECT id FROM s WHERE flag = 1);",
+ want: 1000,
+ },
+ {
+ // The lookup into the materialized subquery prints no loops.
+ fixture: "mariadb_delete_semijoin_materialization.json",
+ statement: "DELETE FROM s WHERE id IN (SELECT s_id FROM big WHERE v = 0);",
+ want: 100,
+ },
+ {
+ // min(10000, 1000) for the target big.
+ fixture: "mariadb_update_straight_join.json",
+ statement: "UPDATE big STRAIGHT_JOIN s ON big.s_id = s.id SET big.v = big.v + 1 WHERE s.flag = 1;",
+ want: 1000,
+ },
+ {
+ // big: min(1000, 1000); s: min(10, 1000).
+ fixture: "mariadb_update_two_targets.json",
+ statement: "UPDATE big JOIN s ON big.s_id = s.id SET big.v = 1, s.flag = 1 WHERE s.flag = 1;",
+ want: 1010,
+ },
+ {
+ fixture: "mariadb_update_join_unqualified_column.json",
+ statement: "UPDATE big AS b JOIN s AS x ON b.s_id = x.id SET v = 1 WHERE x.flag = 1;",
+ want: 1000,
+ },
+ {
+ // Either joined table may own each unqualified column: 2 * 1000.
+ fixture: "mariadb_update_join_unqualified_column.json",
+ statement: "UPDATE big AS b JOIN s AS x ON b.s_id = x.id SET v = 1, flag = 0 WHERE x.flag = 1;",
+ want: 2000,
+ },
+ {
+ fixture: "mariadb_delete_impossible_where.json",
+ statement: "DELETE FROM t WHERE 1 = 0;",
+ want: 0,
+ },
+ {
+ // 1000 rows filtered to 14.3%.
+ fixture: "mariadb_insert_select_filter.json",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE d = 3;",
+ want: 143,
+ },
+ {
+ fixture: "mariadb_insert_select_join.json",
+ statement: "INSERT INTO big2 SELECT big.* FROM s JOIN big ON big.s_id = s.id WHERE s.flag = 1;",
+ want: 1000,
+ },
+ {
+ fixture: "mariadb_insert_select_union_all.json",
+ statement: "INSERT INTO t2 SELECT * FROM t WHERE c < 10 UNION ALL SELECT * FROM t WHERE c >= 10;",
+ want: 1000,
+ },
+ {
+ fixture: "mariadb_insert_select_block_nl_join.json",
+ statement: "INSERT INTO t2 SELECT t.* FROM t JOIN s ON t.d = s.flag WHERE s.id < 50;",
+ want: 49000,
+ },
+ {
+ fixture: "mariadb_insert_select_order_limit.json",
+ statement: "INSERT INTO t2 SELECT * FROM t ORDER BY d LIMIT 50;",
+ want: 50,
+ },
+ {
+ fixture: "mariadb_insert_select_window.json",
+ statement: "INSERT INTO t2 (id, c, d, grp, v) SELECT id, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY id), d, grp, v FROM t WHERE c < 10;",
+ want: 100,
+ },
+ {
+ fixture: "mariadb_insert_select_no_tables.json",
+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a';",
+ want: 1,
+ },
+ {
+ // MySQL prints only the INSERT target for a source that reads no table.
+ fixture: "insert_select_from_dual_not_exists.json",
+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a' FROM DUAL WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE id = 1);",
+ want: 1,
+ },
+ {
+ fixture: "insert_select_union_without_tables.json",
+ statement: "INSERT INTO t2 SELECT 1, 1, 1, 1, 'a' UNION ALL SELECT 2, 2, 2, 2, 'b';",
+ want: 2,
},
} {
t.Run(tc.fixture, func(t *testing.T) {
- plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", tc.fixture))
+ got, err := EstimateAffectedRowsFromExplainJSON(parseSingleStatement(t, tc.statement), readExplainPlanFixture(t, tc.fixture))
require.NoError(t, err)
- rows, found := GetEstimatedAffectedRowsFromExplainJSON(string(plan))
- require.Equal(t, tc.wantFound, found)
- require.Equal(t, tc.wantRows, rows)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestTableFreeSelectRows(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ want int
+ wantOK bool
+ }{
+ {statement: "INSERT INTO t SELECT 1 UNION ALL SELECT 2 UNION SELECT 3", want: 3, wantOK: true},
+ {statement: "INSERT INTO t SELECT 1 INTERSECT SELECT 1 FROM DUAL", want: 1, wantOK: true},
+ {statement: "INSERT INTO t SELECT 1 EXCEPT SELECT 2", want: 1, wantOK: true},
+ {statement: "INSERT INTO t SELECT 1 UNION ALL SELECT id FROM s"},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ insert, ok := parseSingleStatement(t, tc.statement).(*ast.InsertStmt)
+ require.True(t, ok)
+ got, ok := tableFreeSelectRows(insert.Select)
+ require.Equal(t, tc.wantOK, ok)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestDMLTargetCount(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ want int
+ }{
+ {statement: "UPDATE t SET a = 1, b = 2 WHERE id = 1", want: 1},
+ {statement: "UPDATE a JOIN b ON a.id = b.id SET a.x = 1, b.y = 1", want: 2},
+ {statement: "UPDATE a JOIN b ON a.id = b.id SET x = 1, y = 2, z = 3", want: 2},
+ {statement: "DELETE a, b FROM a JOIN b ON a.id = b.id", want: 2},
+ {statement: "DELETE FROM t WHERE id = 1", want: 1},
+ {statement: "INSERT INTO t SELECT * FROM s", want: 1},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ require.Equal(t, tc.want, DMLTargetCount(parseSingleStatement(t, tc.statement), 2))
+ })
+ }
+}
+
+func TestCapAffectedRowsByLimit(t *testing.T) {
+ for _, tc := range []struct {
+ statement string
+ rows float64
+ want int64
+ }{
+ {statement: "INSERT INTO t2 SELECT t.id FROM t, big, big AS big3", rows: 1e30, want: math.MaxInt64},
+ {statement: "INSERT INTO t2 SELECT t.id FROM t, big, big AS big3 LIMIT 10", rows: 1e30, want: 10},
+ {statement: "DELETE FROM t WHERE id > 0 LIMIT 5", rows: 2.5, want: 3},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ require.Equal(t, tc.want, CapAffectedRowsByLimit(parseSingleStatement(t, tc.statement), tc.rows))
})
}
}
-func TestGetEstimatedAffectedRowsFromExplainJSONInvalidInput(t *testing.T) {
- for _, plan := range []string{"", "not json", "[]", `{"query_block":{}}`} {
- rows, found := GetEstimatedAffectedRowsFromExplainJSON(plan)
- require.False(t, found, plan)
- require.Zero(t, rows, plan)
+func TestEstimateAffectedRowsFromExplainJSONError(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ plan string
+ statement string
+ wantErr string
+ }{
+ {
+ // MySQL 8.0 omits insert_from when the SELECT computes window functions.
+ name: "insert without a source plan",
+ plan: readExplainPlanFixture(t, "insert_select_window_no_source.json"),
+ statement: "INSERT INTO t2 (id, c, d, grp, v) SELECT id, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY id), d, grp, v FROM t WHERE c < 10;",
+ wantErr: `table "t2" in the plan has no row estimate`,
+ },
+ {
+ name: "MariaDB delete of all rows",
+ plan: readExplainPlanFixture(t, "mariadb_delete_all_rows.json"),
+ statement: "DELETE FROM big;",
+ wantErr: "the plan has no row estimate: Deleting all rows",
+ },
+ {
+ name: "format version 2",
+ plan: `{"query": "/* select#1 */ delete from t", "operation": "Delete from t", "access_type": "table", "estimated_rows": 1000}`,
+ statement: "DELETE FROM t;",
+ wantErr: "no query_block",
+ },
+ {
+ name: "unrecognized plan node",
+ plan: `{"query_block": {"select_id": 1, "future_operation": {"table": {"delete": true, "table_name": "t", "rows_examined_per_scan": 10}}}}`,
+ statement: "DELETE FROM t;",
+ wantErr: "unrecognized EXPLAIN FORMAT=JSON plan node",
+ },
+ {
+ name: "not JSON",
+ plan: "-> Delete from t",
+ statement: "DELETE FROM t;",
+ wantErr: "failed to parse EXPLAIN FORMAT=JSON output",
+ },
+ {
+ name: "unsupported statement",
+ plan: readExplainPlanFixture(t, "select_only.json"),
+ statement: "SELECT * FROM target_table;",
+ wantErr: "unsupported statement type",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := EstimateAffectedRowsFromExplainJSON(parseSingleStatement(t, tc.statement), tc.plan)
+ require.ErrorContains(t, err, tc.wantErr)
+ })
+ }
+}
+
+func TestEstimateAffectedRowsFromOceanBaseExplainJSON(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ plan string
+ want int64
+ }{
+ {
+ name: "root operator",
+ plan: `{"ID":0,"OPERATOR":"UPDATE","NAME":"","EST.ROWS":1000,"EST.TIME(us)":7680,"output":"","CHILD_1":{"ID":1,"OPERATOR":"TABLE RANGE SCAN","NAME":"dba_test_1","EST.ROWS":1200,"EST.TIME(us)":91}}`,
+ want: 1000,
+ },
+ {
+ name: "fractional estimate",
+ plan: `{"ID":0,"OPERATOR":"DELETE","NAME":"","EST.ROWS":2.6,"EST.TIME(us)":20}`,
+ want: 3,
+ },
+ {
+ name: "largest child operator without a root operator",
+ plan: `{"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t1","EST.ROWS":40},"CHILD_2":{"ID":2,"OPERATOR":"TABLE FULL SCAN","NAME":"t2","EST.ROWS":70}}`,
+ want: 70,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := EstimateAffectedRowsFromOceanBaseExplainJSON(tc.plan)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ })
+ }
+
+ for _, tc := range []struct {
+ name string
+ plan string
+ wantErr string
+ }{
+ {
+ name: "root operator without an estimate",
+ plan: `{"ID":0,"OPERATOR":"UPDATE","NAME":""}`,
+ wantErr: `operator "UPDATE" has no EST.ROWS`,
+ },
+ {
+ name: "child operator without an estimate",
+ plan: `{"CHILD_1":{"ID":1,"OPERATOR":"TABLE FULL SCAN","NAME":"t1"}}`,
+ wantErr: `operator "TABLE FULL SCAN" has no EST.ROWS`,
+ },
+ {
+ name: "no operator",
+ plan: `{}`,
+ wantErr: "the plan has no operator",
+ },
+ {
+ name: "empty output",
+ plan: "",
+ wantErr: "failed to parse EXPLAIN FORMAT=JSON output",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := EstimateAffectedRowsFromOceanBaseExplainJSON(tc.plan)
+ require.ErrorContains(t, err, tc.wantErr)
+ })
}
}
+
+func readExplainPlanFixture(t *testing.T, fixture string) string {
+ t.Helper()
+ plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", fixture))
+ require.NoError(t, err)
+ return string(plan)
+}
+
+func parseSingleStatement(t *testing.T, statement string) ast.Node {
+ t.Helper()
+ list, err := ParseMySQL(statement)
+ require.NoError(t, err)
+ require.Len(t, list.Items, 1)
+ return list.Items[0]
+}
diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.go
index 940b6eee3a..9c1ae0df7b 100644
--- a/backend/plugin/parser/mysql/resource_change.go
+++ b/backend/plugin/parser/mysql/resource_change.go
@@ -6,7 +6,6 @@ import (
"github.com/bytebase/omni/mysql/ast"
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -24,7 +23,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
changedResources := model.NewChangedResources(dbMetadata)
var dmlCount, insertCount int
- var sampleDMLs []string
+ var dmlStatements []string
// addObjectDatabase records a database-only write target for a non-table object DDL
// (view/function/routine/trigger) ONLY when the object's own name carries an explicit
@@ -109,19 +108,20 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
insertCount += len(n.Values)
continue
}
- dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
+ // INSERT ... SET inserts exactly one row, and EXPLAIN gives no estimate for it.
+ if len(n.SetList) > 0 {
+ insertCount++
+ continue
}
+ dmlCount++
+ dmlStatements = append(dmlStatements, omniStatementText(omniAST))
case *ast.UpdateStmt:
for _, resource := range extractTableExprs(n.Tables, currentDatabase) {
changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)
}
dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
- }
+ dmlStatements = append(dmlStatements, omniStatementText(omniAST))
case *ast.DeleteStmt:
for _, resource := range extractTableExprs(n.Tables, currentDatabase) {
@@ -131,9 +131,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)
}
dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
- }
+ dmlStatements = append(dmlStatements, omniStatementText(omniAST))
// Non-table object DDL: gate by the object's own explicit database qualifier only.
// CREATE TRIGGER is intentionally skipped — its AST name is unqualified (a bare string);
@@ -161,7 +159,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
return &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: sampleDMLs,
+ DMLStatements: dmlStatements,
DMLCount: dmlCount,
InsertCount: insertCount,
}, nil
diff --git a/backend/plugin/parser/mysql/resource_change_test.go b/backend/plugin/parser/mysql/resource_change_test.go
index a983e521a1..a083913bb2 100644
--- a/backend/plugin/parser/mysql/resource_change_test.go
+++ b/backend/plugin/parser/mysql/resource_change_test.go
@@ -16,6 +16,7 @@ func TestExtractChangedResources(t *testing.T) {
ALTER TABLE t1 ADD COLUMN c1 INT;
RENAME TABLE t1 TO t2;
INSERT INTO t1 (c1) VALUES (1), (5);
+ INSERT INTO t1 SET c1 = 7;
UPDATE t1 SET c1 = 5;
CREATE PROCEDURE getUser(id INT) SELECT * FROM hello WHERE uid = id;
`
@@ -38,11 +39,11 @@ func TestExtractChangedResources(t *testing.T) {
)
want := &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: []string{
+ DMLStatements: []string{
"UPDATE t1 SET c1 = 5;",
},
DMLCount: 1,
- InsertCount: 2,
+ InsertCount: 3,
}
stmts, err := base.ParseStatements(storepb.Engine_MYSQL, statement)
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.json b/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.json
new file mode 100644
index 0000000000..68d2c466da
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/delete_cte_semijoin_materialization_scan.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"122.70"},"nested_loop":[{"table":{"table_name":"<subquery2>","access_type":"ALL","materialized_from_subquery":{"using_temporary_table":true,"query_block":{"table":{"table_name":"x","access_type":"ALL","rows_examined_per_scan":10,"rows_produced_per_join":10,"filtered":"100.00","cost_info":{"read_cost":"2.62","eval_cost":"1.00","prefix_cost":"3.62","data_read_per_join":"160"},"used_columns":["id"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"10.25"},"table":{"table_name":"s","access_type":"ALL","rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}}}}}}},{"table":{"delete":true,"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["<subquery2>.id"],"rows_examined_per_scan":113,"rows_produced_per_join":113,"filtered":"100.00","cost_info":{"read_cost":"2.77","eval_cost":"11.33","prefix_cost":"122.70","data_read_per_join":"1K"},"used_columns":["id","s_id","v"]}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.json
new file mode 100644
index 0000000000..7c9f98664b
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_buffer_result.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"451.75"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"},"insert_from":{"buffer_result":{"using_temporary_table":true,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"t2","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.t.id"],"rows_examined_per_scan":1,"rows_produced_per_join":1000,"filtered":"100.00","not_exists":true,"using_index":true,"cost_info":{"read_cost":"250.00","eval_cost":"100.00","prefix_cost":"451.75","data_read_per_join":"101K"},"used_columns":["id"],"attached_condition":"<if>(found_match(t2), (`probe`.`t2`.`id` is null), true)"}}]}}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.json
new file mode 100644
index 0000000000..3c0c012135
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_filter.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"101.75"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.75","eval_cost":"10.00","prefix_cost":"101.75","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d` = 3)"}}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.json
new file mode 100644
index 0000000000..c61d121fa7
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_from_dual_not_exists.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"table":{"insert":true,"table_name":"t2","access_type":"ALL","attached_subqueries":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"message":"no matching row in const table"}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.json
new file mode 100644
index 0000000000..fa8e537378
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_impossible_where.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"message":"Impossible WHERE"}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.json
new file mode 100644
index 0000000000..0c9c2cda0a
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"406.80"},"table":{"insert":true,"table_name":"big2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["probe.s.id"],"rows_examined_per_scan":113,"rows_produced_per_join":1133,"filtered":"100.00","cost_info":{"read_cost":"283.25","eval_cost":"113.30","prefix_cost":"406.80","data_read_per_join":"17K"},"used_columns":["id","s_id","v"]}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.json
new file mode 100644
index 0000000000..0678ec504d
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_parenthesized_limit.json
@@ -0,0 +1 @@
+{"query_block":{"unary_result":{"using_temporary_table":true,"select_id":2,"table_name":"<ordered/limited1>","access_type":"ALL","using_filesort":true,"query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"101.25"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"},"insert_from":{"ordering_operation":{"using_filesort":true,"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.25","eval_cost":"100.00","prefix_cost":"101.25","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}}}}}]}}}
\ No newline at end of file
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.json
new file mode 100644
index 0000000000..3aaf15e7e0
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_semijoin_materialization_lookup.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"207.37"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"<subquery2>","access_type":"eq_ref","key":"<auto_distinct_key>","key_length":"4","ref":["probe.t.grp"],"rows_examined_per_scan":1,"materialized_from_subquery":{"using_temporary_table":true,"query_block":{"table":{"table_name":"x","access_type":"ALL","rows_examined_per_scan":10,"rows_produced_per_join":10,"filtered":"100.00","cost_info":{"read_cost":"2.62","eval_cost":"1.00","prefix_cost":"3.62","data_read_per_join":"160"},"used_columns":["id"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"10.25"},"table":{"table_name":"s","access_type":"ALL","rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}}}}}}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.json
new file mode 100644
index 0000000000..e724d9f532
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_all.json
@@ -0,0 +1 @@
+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":3,"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"45.26"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","used_key_parts":["c"],"key_length":"4","rows_examined_per_scan":100,"rows_produced_per_join":100,"filtered":"100.00","index_condition":"(`probe`.`t`.`c` < 10)","cost_info":{"read_cost":"35.26","eval_cost":"10.00","prefix_cost":"45.26","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"]}}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.75"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":899,"filtered":"90.00","cost_info":{"read_cost":"11.75","eval_cost":"90.00","prefix_cost":"101.75","data_read_per_join":"91K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c` >= 10)"}}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.json
new file mode 100644
index 0000000000..ed37ba9574
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_except.json
@@ -0,0 +1 @@
+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":5,"table_name":"<union1,4>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"1.60"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"small","access_type":"index","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","rows_examined_per_scan":3,"rows_produced_per_join":3,"filtered":"100.00","using_index":true,"cost_info":{"read_cost":"0.25","eval_cost":"0.30","prefix_cost":"0.55","data_read_per_join":"48"},"used_columns":["id"]}},{"table":{"table_name":"t","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.small.id"],"rows_examined_per_scan":1,"rows_produced_per_join":3,"filtered":"100.00","cost_info":{"read_cost":"0.75","eval_cost":"0.30","prefix_cost":"1.60","data_read_per_join":"312"},"used_columns":["id","c","d","grp","v"]}}]}}},{"except_result":{"using_temporary_table":true,"select_id":4,"table_name":"<except2,3>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":500,"filtered":"50.00","cost_info":{"read_cost":"51.25","eval_cost":"50.00","prefix_cost":"101.25","data_read_per_join":"50K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c`<50)"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.25","eval_cost":"10.00","prefix_cost":"101.25","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d`=1)"}}}]}}]}}}
\ No newline at end of file
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.json
new file mode 100644
index 0000000000..9a9372c62a
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_intersect.json
@@ -0,0 +1 @@
+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":5,"table_name":"<union1,4>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"cost_info":{"query_cost":"1.60"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"small","access_type":"index","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","rows_examined_per_scan":3,"rows_produced_per_join":3,"filtered":"100.00","using_index":true,"cost_info":{"read_cost":"0.25","eval_cost":"0.30","prefix_cost":"0.55","data_read_per_join":"48"},"used_columns":["id"]}},{"table":{"table_name":"t","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.small.id"],"rows_examined_per_scan":1,"rows_produced_per_join":3,"filtered":"100.00","cost_info":{"read_cost":"0.75","eval_cost":"0.30","prefix_cost":"1.60","data_read_per_join":"312"},"used_columns":["id","c","d","grp","v"]}}]}}},{"intersect_result":{"using_temporary_table":true,"select_id":4,"table_name":"<intersect2,3>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"rows_examined_per_scan":1000,"rows_produced_per_join":500,"filtered":"50.00","cost_info":{"read_cost":"51.25","eval_cost":"50.00","prefix_cost":"101.25","data_read_per_join":"50K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`c`<50)"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":3,"cost_info":{"query_cost":"101.25"},"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":100,"filtered":"10.00","cost_info":{"read_cost":"91.25","eval_cost":"10.00","prefix_cost":"101.25","data_read_per_join":"10K"},"used_columns":["id","c","d","grp","v"],"attached_condition":"(`probe`.`t`.`d`=1)"}}}]}}]}}}
\ No newline at end of file
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.json
new file mode 100644
index 0000000000..2d13676ced
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_union_without_tables.json
@@ -0,0 +1 @@
+{"query_block":{"union_result":{"using_temporary_table":true,"select_id":3,"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"dependent":false,"cacheable":true,"query_block":{"select_id":1,"table":{"insert":true,"table_name":"t2","access_type":"ALL"}}},{"dependent":false,"cacheable":true,"query_block":{"select_id":2,"message":"No tables used"}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.json
new file mode 100644
index 0000000000..20a033bf09
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_select_window_no_source.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"145.26"},"table":{"insert":true,"select_id":1,"table_name":"t2","access_type":"ALL"}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.json b/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.json
new file mode 100644
index 0000000000..021181aa2c
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/insert_table.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"101.75"},"table":{"insert":true,"table_name":"t2","access_type":"ALL"},"insert_from":{"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.json
new file mode 100644
index 0000000000..1a38f4cc72
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_all_rows.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"table":{"message":"Deleting all rows"}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.json
new file mode 100644
index 0000000000..d1b86a015c
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_impossible_where.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"table":{"message":"Impossible WHERE"}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.json
new file mode 100644
index 0000000000..b2717d8279
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":0.1878414,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":0.1606866,"filtered":100}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.json
new file mode 100644
index 0000000000..b2717d8279
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":0.1878414,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":0.1606866,"filtered":100}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.json
new file mode 100644
index 0000000000..e22ea5820f
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_delete_semijoin_materialization.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":3.3635308,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":100}},{"table":{"table_name":"<subquery2>","access_type":"eq_ref","possible_keys":["distinct_key"],"key":"distinct_key","key_length":"4","used_key_parts":["s_id"],"ref":["func"],"rows":1,"filtered":100,"materialized":{"unique":1,"query_block":{"select_id":2,"nested_loop":[{"table":{"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"loops":1,"rows":10000,"cost":1.6588056,"filtered":100,"attached_condition":"big.v = 0"}}]}}}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.json
new file mode 100644
index 0000000000..dec3d3d35a
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_block_nl_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":4.741674304,"nested_loop":[{"table":{"table_name":"s","access_type":"range","possible_keys":["PRIMARY"],"key":"PRIMARY","key_length":"4","used_key_parts":["id"],"loops":1,"rows":49,"cost":0.01183867,"filtered":100,"attached_condition":"s.`id` < 50"}},{"block-nl-join":{"table":{"table_name":"t","access_type":"ALL","loops":49,"rows":1000,"cost":4.729835634,"filtered":100},"buffer_type":"flat","buffer_size":"348","join_type":"BNL","attached_condition":"t.d = s.flag"}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.json
new file mode 100644
index 0000000000..9d04f7c04d
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_filter.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":0.1764192,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","loops":1,"rows":1000,"cost":0.1764192,"filtered":14.30000019,"attached_condition":"t.d = 3"}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.json
new file mode 100644
index 0000000000..d729014ba7
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.json
new file mode 100644
index 0000000000..6d7280a4e0
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_no_tables.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"table":{"message":"No tables used"}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.json
new file mode 100644
index 0000000000..c2b88282d3
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_order_limit.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":0.1764192,"nested_loop":[{"read_sorted_file":{"filesort":{"sort_key":"t.d","table":{"table_name":"t","access_type":"ALL","loops":1,"rows":1000,"cost":0.1764192,"filtered":100}}}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.json
new file mode 100644
index 0000000000..36bcf5c677
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_union_all.json
@@ -0,0 +1 @@
+{"query_block":{"union_result":{"table_name":"<union1,2>","access_type":"ALL","query_specifications":[{"query_block":{"select_id":1,"cost":0.10505784,"nested_loop":[{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","key_length":"4","used_key_parts":["c"],"loops":1,"rows":100,"cost":0.10505784,"filtered":100,"index_condition":"t.c < 10"}}]}},{"query_block":{"select_id":2,"operation":"UNION","cost":0.1764192,"nested_loop":[{"table":{"table_name":"t","access_type":"ALL","possible_keys":["idx_c"],"loops":1,"rows":1000,"cost":0.1764192,"filtered":90,"attached_condition":"t.c >= 10"}}]}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.json
new file mode 100644
index 0000000000..987f70c55c
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_insert_select_window.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":0.10505784,"window_functions_computation":{"sorts":[{"filesort":{"sort_key":"t.grp, t.`id`"}}],"temporary_table":{"nested_loop":[{"table":{"table_name":"t","access_type":"range","possible_keys":["idx_c"],"key":"idx_c","key_length":"4","used_key_parts":["c"],"loops":1,"rows":100,"cost":0.10505784,"filtered":100,"index_condition":"t.c < 10"}}]}}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.json
new file mode 100644
index 0000000000..cc51e45541
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_all_rows.json
@@ -0,0 +1,22 @@
+{
+ "query_block": {
+ "select_id": 1,
+ "cost": 1.47862472,
+ "nested_loop": [
+ {
+ "table": {
+ "table_name": "big",
+ "access_type": "index",
+ "key": "idx_s",
+ "key_length": "4",
+ "used_key_parts": ["s_id"],
+ "loops": 1,
+ "rows": 10000,
+ "cost": 1.47862472,
+ "filtered": 100,
+ "using_index": true
+ }
+ }
+ ]
+ }
+}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.json
new file mode 100644
index 0000000000..aa661e48bf
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_select_for_delete_range_filter.json
@@ -0,0 +1,24 @@
+{
+ "query_block": {
+ "select_id": 1,
+ "cost": 0.10505784,
+ "nested_loop": [
+ {
+ "table": {
+ "table_name": "t",
+ "access_type": "range",
+ "possible_keys": ["idx_c"],
+ "key": "idx_c",
+ "key_length": "4",
+ "used_key_parts": ["c"],
+ "loops": 1,
+ "rows": 100,
+ "cost": 0.10505784,
+ "filtered": 14.30000019,
+ "index_condition": "t.c < 10",
+ "attached_condition": "t.d = 3"
+ }
+ }
+ ]
+ }
+}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.json
new file mode 100644
index 0000000000..c54018b34f
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_join_unqualified_column.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"x","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"x.flag = 1"}},{"table":{"table_name":"b","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.x.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.json
new file mode 100644
index 0000000000..d36b956fe1
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_straight_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":10.6040248,"nested_loop":[{"table":{"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"loops":1,"rows":10000,"cost":1.6588056,"filtered":100}},{"table":{"table_name":"s","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","key_length":"4","used_key_parts":["id"],"ref":["probe.big.s_id"],"loops":10000,"rows":1,"cost":8.9452192,"filtered":10,"attached_condition":"s.flag = 1"}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.json b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.json
new file mode 100644
index 0000000000..d729014ba7
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mariadb_update_two_targets.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost":1.0526188,"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"loops":1,"rows":100,"cost":0.0271548,"filtered":10,"attached_condition":"s.flag = 1"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","key_length":"4","used_key_parts":["s_id"],"ref":["probe.s.id"],"loops":10,"rows":100,"cost":1.025464,"filtered":100}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.json b/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.json
new file mode 100644
index 0000000000..250ed37d3c
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/mysql57_delete_all_rows.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"table":{"delete":true,"table_name":"big","access_type":"ALL","rows_examined_per_scan":10195,"filtered":"100.00","message":"Deleting all rows"}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.json
new file mode 100644
index 0000000000..560946e571
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/replace_select_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"406.80"},"table":{"replace":true,"table_name":"big2","access_type":"ALL"},"insert_from":{"nested_loop":[{"table":{"table_name":"s","access_type":"ALL","possible_keys":["PRIMARY"],"rows_examined_per_scan":100,"rows_produced_per_join":10,"filtered":"10.00","cost_info":{"read_cost":"9.25","eval_cost":"1.00","prefix_cost":"10.25","data_read_per_join":"80"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}},{"table":{"table_name":"big","access_type":"ref","possible_keys":["idx_s"],"key":"idx_s","used_key_parts":["s_id"],"key_length":"4","ref":["probe.s.id"],"rows_examined_per_scan":113,"rows_produced_per_join":1133,"filtered":"100.00","cost_info":{"read_cost":"283.25","eval_cost":"113.30","prefix_cost":"406.80","data_read_per_join":"17K"},"used_columns":["id","s_id","v"]}}]}}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.json b/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.json
new file mode 100644
index 0000000000..400efad569
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/select_for_update_filter.json
@@ -0,0 +1,25 @@
+{
+ "query_block": {
+ "select_id": 1,
+ "cost_info": {
+ "query_cost": "101.75"
+ },
+ "table": {
+ "table_name": "t",
+ "access_type": "ALL",
+ "rows_examined_per_scan": 1000,
+ "rows_produced_per_join": 100,
+ "filtered": "10.00",
+ "cost_info": {
+ "read_cost": "91.75",
+ "eval_cost": "10.00",
+ "prefix_cost": "101.75",
+ "data_read_per_join": "10K"
+ },
+ "used_columns": [
+ "d"
+ ],
+ "attached_condition": "(`probe`.`t`.`d` = 3)"
+ }
+ }
+}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.json
new file mode 100644
index 0000000000..855666c036
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_derived_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"3601.75"},"nested_loop":[{"table":{"update":true,"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","c","d","grp","v"]}},{"table":{"table_name":"g","access_type":"ref","possible_keys":["<auto_key0>"],"key":"<auto_key0>","used_key_parts":["grp"],"key_length":"4","ref":["probe.t.grp"],"rows_examined_per_scan":10,"rows_produced_per_join":10000,"filtered":"100.00","cost_info":{"read_cost":"2500.00","eval_cost":"1000.00","prefix_cost":"3601.75","data_read_per_join":"234K"},"used_columns":["grp","cnt"],"materialized_from_subquery":{"using_temporary_table":true,"dependent":false,"cacheable":true,"query_block":{"select_id":2,"cost_info":{"query_cost":"101.75"},"grouping_operation":{"using_temporary_table":true,"using_filesort":false,"table":{"table_name":"t","access_type":"ALL","rows_examined_per_scan":1000,"rows_produced_per_join":1000,"filtered":"100.00","cost_info":{"read_cost":"1.75","eval_cost":"100.00","prefix_cost":"101.75","data_read_per_join":"101K"},"used_columns":["id","grp"]}}}}}}]}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.json
new file mode 100644
index 0000000000..fa8e537378
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_impossible_where.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"message":"Impossible WHERE"}}
diff --git a/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.json b/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.json
new file mode 100644
index 0000000000..5d6aff7284
--- /dev/null
+++ b/backend/plugin/parser/mysql/test-data/explain-plan/update_target_first_straight_join.json
@@ -0,0 +1 @@
+{"query_block":{"select_id":1,"cost_info":{"query_cost":"5122.75"},"nested_loop":[{"table":{"update":true,"table_name":"big","access_type":"ALL","possible_keys":["idx_s"],"rows_examined_per_scan":11330,"rows_produced_per_join":11330,"filtered":"100.00","cost_info":{"read_cost":"24.25","eval_cost":"1133.00","prefix_cost":"1157.25","data_read_per_join":"177K"},"used_columns":["id","s_id","v"]}},{"table":{"table_name":"s","access_type":"eq_ref","possible_keys":["PRIMARY"],"key":"PRIMARY","used_key_parts":["id"],"key_length":"4","ref":["probe.big.s_id"],"rows_examined_per_scan":1,"rows_produced_per_join":1133,"filtered":"10.00","cost_info":{"read_cost":"2832.50","eval_cost":"113.30","prefix_cost":"5122.75","data_read_per_join":"8K"},"used_columns":["id","flag"],"attached_condition":"(`probe`.`s`.`flag` = 1)"}}]}}
diff --git a/backend/plugin/parser/pg/explain_plan.go b/backend/plugin/parser/pg/explain_plan.go
new file mode 100644
index 0000000000..4a4ee8fbae
--- /dev/null
+++ b/backend/plugin/parser/pg/explain_plan.go
@@ -0,0 +1,112 @@
+package pg
+
+import (
+ "encoding/json"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
+)
+
+type explainPlanNode struct {
+ NodeType string `json:"Node Type"`
+ ParentRelationship string `json:"Parent Relationship"`
+ PlanRows *float64 `json:"Plan Rows"`
+ Plans []explainPlanNode `json:"Plans"`
+}
+
+// GetEstimatedAffectedRowsFromExplainJSON returns the planner's estimate of the rows modified
+// by the INSERT, UPDATE, DELETE, and MERGE operations in `EXPLAIN (FORMAT JSON)` output. It
+// sums every ModifyTable node, including those of data-modifying CTEs, and returns an error
+// when the plan has none or one cannot be interpreted.
+func GetEstimatedAffectedRowsFromExplainJSON(plan string) (int64, error) {
+ roots, err := parseExplainJSON(plan)
+ if err != nil {
+ return 0, err
+ }
+
+ var total float64
+ modifyTableCount := 0
+ var walk func(node *explainPlanNode) error
+ walk = func(node *explainPlanNode) error {
+ if node.NodeType == "ModifyTable" {
+ rows, err := getModifiedRows(node)
+ if err != nil {
+ return err
+ }
+ total += rows
+ modifyTableCount++
+ }
+ for i := range node.Plans {
+ if err := walk(&node.Plans[i]); err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+ for _, root := range roots {
+ if err := walk(root); err != nil {
+ return 0, err
+ }
+ }
+ if modifyTableCount == 0 {
+ return 0, errors.New("the plan has no ModifyTable node")
+ }
+ return common.RoundRows(total), nil
+}
+
+// GetEstimatedInsertedRowsFromExplainJSON returns the planner's estimate of the rows a top-level
+// INSERT adds in `EXPLAIN (FORMAT JSON)` output, without the rows its data-modifying CTEs change.
+func GetEstimatedInsertedRowsFromExplainJSON(plan string) (int64, error) {
+ roots, err := parseExplainJSON(plan)
+ if err != nil {
+ return 0, err
+ }
+ if len(roots) != 1 || roots[0].NodeType != "ModifyTable" {
+ return 0, errors.New("the plan root is not a ModifyTable node")
+ }
+ rows, err := getModifiedRows(roots[0])
+ if err != nil {
+ return 0, err
+ }
+ return common.RoundRows(rows), nil
+}
+
+// parseExplainJSON returns the root plan node of each statement in `EXPLAIN (FORMAT JSON)` output.
+func parseExplainJSON(plan string) ([]*explainPlanNode, error) {
+ var statements []struct {
+ Plan *explainPlanNode `json:"Plan"`
+ }
+ if err := json.Unmarshal([]byte(plan), &statements); err != nil {
+ return nil, errors.Wrap(err, "failed to parse the EXPLAIN (FORMAT JSON) output")
+ }
+ var roots []*explainPlanNode
+ for _, statement := range statements {
+ if statement.Plan != nil {
+ roots = append(roots, statement.Plan)
+ }
+ }
+ return roots, nil
+}
+
+// getModifiedRows reads the estimate from the subplans feeding a ModifyTable node. PostgreSQL 14
+// and later label its only subplan "Outer"; earlier versions label one "Member" per target table.
+func getModifiedRows(modifyTable *explainPlanNode) (float64, error) {
+ var rows float64
+ found := false
+ for _, child := range modifyTable.Plans {
+ switch child.ParentRelationship {
+ case "Outer", "Member":
+ if child.PlanRows == nil {
+ return 0, errors.Errorf("the %s node feeding ModifyTable has no Plan Rows", child.NodeType)
+ }
+ rows += *child.PlanRows
+ found = true
+ default:
+ }
+ }
+ if !found {
+ return 0, errors.New("the ModifyTable node has no subplan feeding it")
+ }
+ return rows, nil
+}
diff --git a/backend/plugin/parser/pg/explain_plan_test.go b/backend/plugin/parser/pg/explain_plan_test.go
new file mode 100644
index 0000000000..6438d58e42
--- /dev/null
+++ b/backend/plugin/parser/pg/explain_plan_test.go
@@ -0,0 +1,100 @@
+package pg
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// The fixtures are real `EXPLAIN (FORMAT JSON)` outputs captured from PostgreSQL 17 after ANALYZE:
+// t has 1000 rows, s has 100 rows (10 flagged), big has 100 rows per s id, and parted and the
+// inheritance parent inh_parent each hold 1000 rows.
+func TestGetEstimatedAffectedRowsFromExplainJSON(t *testing.T) {
+ for _, tc := range []struct {
+ fixture string
+ wantRows int64
+ }{
+ {fixture: "update.json", wantRows: 100},
+ {fixture: "delete_using_join.json", wantRows: 1000},
+ {fixture: "update_from_join.json", wantRows: 1000},
+ {fixture: "delete_in_subquery.json", wantRows: 1000},
+ {fixture: "update_correlated_set_subquery.json", wantRows: 100},
+ {fixture: "update_initplan.json", wantRows: 3333},
+ {fixture: "update_partitioned.json", wantRows: 100},
+ {fixture: "delete_inheritance_parent.json", wantRows: 101},
+ {fixture: "insert_select.json", wantRows: 100},
+ {fixture: "insert_select_on_conflict_do_nothing.json", wantRows: 1000},
+ {fixture: "insert_values_on_conflict.json", wantRows: 2},
+ {fixture: "merge.json", wantRows: 10000},
+ {fixture: "cte_delete_select.json", wantRows: 5000},
+ // The CTE deletes 5000 rows and the outer INSERT inserts them again.
+ {fixture: "cte_delete_insert.json", wantRows: 10000},
+ } {
+ t.Run(tc.fixture, func(t *testing.T) {
+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(readExplainPlanFixture(t, tc.fixture))
+ require.NoError(t, err)
+ require.Equal(t, tc.wantRows, rows)
+ })
+ }
+}
+
+func TestGetEstimatedInsertedRowsFromExplainJSON(t *testing.T) {
+ for _, tc := range []struct {
+ fixture string
+ wantRows int64
+ }{
+ {fixture: "insert_select.json", wantRows: 100},
+ // The outer INSERT adds the 5000 rows its CTE deletes.
+ {fixture: "cte_delete_insert.json", wantRows: 5000},
+ } {
+ t.Run(tc.fixture, func(t *testing.T) {
+ rows, err := GetEstimatedInsertedRowsFromExplainJSON(readExplainPlanFixture(t, tc.fixture))
+ require.NoError(t, err)
+ require.Equal(t, tc.wantRows, rows)
+ })
+ }
+
+ _, err := GetEstimatedInsertedRowsFromExplainJSON(readExplainPlanFixture(t, "select.json"))
+ require.EqualError(t, err, "the plan root is not a ModifyTable node")
+}
+
+func TestGetEstimatedAffectedRowsFromExplainJSONMemberSubplans(t *testing.T) {
+ // PostgreSQL 13 and earlier plan a partitioned UPDATE as one ModifyTable subplan per partition.
+ const plan = `[{"Plan": {"Node Type": "ModifyTable", "Operation": "Update", "Plan Rows": 0, "Plans": [
+ {"Node Type": "Seq Scan", "Parent Relationship": "Member", "Relation Name": "parted_p1", "Plan Rows": 50},
+ {"Node Type": "Seq Scan", "Parent Relationship": "Member", "Relation Name": "parted_p2", "Plan Rows": 50}
+ ]}}]`
+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(plan)
+ require.NoError(t, err)
+ require.Equal(t, int64(100), rows)
+}
+
+func TestGetEstimatedAffectedRowsFromExplainJSONError(t *testing.T) {
+ for name, plan := range map[string]string{
+ "select": readExplainPlanFixture(t, "select.json"),
+ "costs off": readExplainPlanFixture(t, "update_costs_off.json"),
+ "empty": "",
+ "not json": "not json",
+ "no statements": "[]",
+ "text format": "Update on t (cost=5.05..13.30 rows=0 width=0)",
+ "modify table without subplan": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0}}]`,
+ "modify table with only an init plan": `[{"Plan": {"Node Type": "ModifyTable", "Plan Rows": 0, "Plans": [
+ {"Node Type": "Result", "Parent Relationship": "InitPlan", "Plan Rows": 1}
+ ]}}]`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ rows, err := GetEstimatedAffectedRowsFromExplainJSON(plan)
+ require.Error(t, err)
+ require.Zero(t, rows)
+ })
+ }
+}
+
+func readExplainPlanFixture(t *testing.T, name string) string {
+ t.Helper()
+ plan, err := os.ReadFile(filepath.Join("test-data", "explain-plan", name))
+ require.NoError(t, err)
+ return string(plan)
+}
diff --git a/backend/plugin/parser/pg/query_antlr.go b/backend/plugin/parser/pg/query_antlr.go
index 31f855d74c..1b7e248006 100644
--- a/backend/plugin/parser/pg/query_antlr.go
+++ b/backend/plugin/parser/pg/query_antlr.go
@@ -1,8 +1,6 @@
package pg
import (
- "strings"
-
"github.com/bytebase/omni/pg/ast"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -43,7 +41,7 @@ func validateQueryANTLR(statement string) (bool, bool, error) {
}
case *ast.ExplainStmt:
- if isExplainAnalyze(n) {
+ if isExplainAnalyzeOmni(n) {
// EXPLAIN ANALYZE executes the query, so it must be a read-only SELECT.
sel, ok := n.Query.(*ast.SelectStmt)
if !ok || isWriteSelect(sel) {
@@ -66,28 +64,12 @@ func validateQueryANTLR(statement string) (bool, bool, error) {
return true, !hasExecute, nil
}
-// isExplainAnalyze checks if an ExplainStmt has the ANALYZE option.
-func isExplainAnalyze(n *ast.ExplainStmt) bool {
- if n.Options == nil {
- return false
- }
- for _, item := range n.Options.Items {
- if de, ok := item.(*ast.DefElem); ok {
- if strings.EqualFold(de.Defname, "analyze") {
- return true
- }
- }
- }
- return false
-}
-
// isWriteSelect reports whether a SelectStmt actually writes — and so must not take
// the read-only editor path. A SELECT writes if it has an INTO target (SELECT ...
// INTO creates a table) or any of its CTE terms is data-modifying.
//
-// This is the unified write-detection primitive for the read-only gate;
-// classifyQueryType (query_type.go) keeps reporting the root statement type for its
-// own consumers — only the detection is shared, not the classifiers' outputs.
+// The read-only gate uses it; classifyQueryType (query_type.go) shares containsWriteCTE
+// so the permission check treats the same SELECTs as DML.
func isWriteSelect(n *ast.SelectStmt) bool {
if n == nil {
return false
diff --git a/backend/plugin/parser/pg/query_test.go b/backend/plugin/parser/pg/query_test.go
index fecc57bda5..30872749b3 100644
--- a/backend/plugin/parser/pg/query_test.go
+++ b/backend/plugin/parser/pg/query_test.go
@@ -38,6 +38,16 @@ func TestValidateSQLForEditor(t *testing.T) {
valid: false,
allQuery: false,
},
+ {
+ sql: `EXPLAIN (ANALYZE off) DELETE FROM t`,
+ valid: true,
+ allQuery: true,
+ },
+ {
+ sql: `EXPLAIN (ANALYZE on) DELETE FROM t`,
+ valid: false,
+ allQuery: false,
+ },
{
sql: `
With t as (
diff --git a/backend/plugin/parser/pg/query_type.go b/backend/plugin/parser/pg/query_type.go
index 5983d96211..ea3821b32f 100644
--- a/backend/plugin/parser/pg/query_type.go
+++ b/backend/plugin/parser/pg/query_type.go
@@ -25,6 +25,10 @@ func classifyQueryType(node ast.Node, allSystems bool) (queryType base.QueryType
if hasOmniIntoClause(n) {
return base.DDL, false
}
+ // A data-modifying CTE writes, so the SELECT needs DML permission.
+ if containsWriteCTE(n) {
+ return base.DML, false
+ }
if allSystems {
return base.SelectInfoSchema, false
}
@@ -120,19 +124,47 @@ func omniIntoClause(n *ast.SelectStmt) *ast.IntoClause {
return omniIntoClause(n.Rarg)
}
-// isExplainAnalyzeOmni checks if an ExplainStmt has the ANALYZE option.
+// isExplainAnalyzeOmni reports whether an ExplainStmt executes its query: its last ANALYZE option
+// is not FALSE, OFF, or 0, the values PostgreSQL reads as false.
func isExplainAnalyzeOmni(n *ast.ExplainStmt) bool {
if n.Options == nil {
return false
}
+ analyze := false
for _, item := range n.Options.Items {
- if de, ok := item.(*ast.DefElem); ok {
- if strings.EqualFold(de.Defname, "analyze") {
- return true
- }
+ de, ok := item.(*ast.DefElem)
+ if !ok || !strings.EqualFold(de.Defname, "analyze") {
+ continue
+ }
+ switch arg := de.Arg.(type) {
+ case *ast.String:
+ analyze = !strings.EqualFold(arg.Str, "false") && !strings.EqualFold(arg.Str, "off")
+ case *ast.Integer:
+ analyze = arg.Ival != 0
+ default:
+ analyze = true
}
}
- return false
+ return analyze
+}
+
+// UnwrapExplainAnalyze returns the statement that an EXPLAIN ANALYZE executes and that statement's text
+// within text, the text of the EXPLAIN. It returns any other node and text unchanged.
+func UnwrapExplainAnalyze(node ast.Node, text string) (ast.Node, string) {
+ explain, ok := node.(*ast.ExplainStmt)
+ if !ok || !isExplainAnalyzeOmni(explain) {
+ return node, text
+ }
+ // The statement runs from its WITH clause, which the location of a SELECT leaves out, to the end
+ // of the EXPLAIN, which its location can also leave out, as for ORDER BY.
+ start := ast.NodeLoc(explain.Query).Start
+ if with := getWithClause(explain.Query); with != nil && with.Loc.Start >= 0 && with.Loc.Start < start {
+ start = with.Loc.Start
+ }
+ if start < 0 || start >= explain.Loc.End || explain.Loc.End > len(text) {
+ return explain.Query, text
+ }
+ return explain.Query, text[start:explain.Loc.End]
}
// classifyExplainedQuery returns the QueryType for the query inside EXPLAIN ANALYZE.
@@ -145,8 +177,11 @@ func classifyExplainedQuery(query ast.Node) base.QueryType {
if hasOmniIntoClause(n) {
return base.DDL
}
+ if containsWriteCTE(n) {
+ return base.DML
+ }
return base.Select
- case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:
+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.MergeStmt:
return base.DML
case *ast.DeclareCursorStmt:
return base.Select
diff --git a/backend/plugin/parser/pg/resource_change.go b/backend/plugin/parser/pg/resource_change.go
index ba5c699c1e..c711498867 100644
--- a/backend/plugin/parser/pg/resource_change.go
+++ b/backend/plugin/parser/pg/resource_change.go
@@ -1,12 +1,12 @@
package pg
import (
+ "slices"
"strings"
"github.com/bytebase/omni/pg/ast"
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -14,7 +14,6 @@ import (
func init() {
base.RegisterExtractChangedResourcesFunc(storepb.Engine_POSTGRES, extractChangedResources)
- base.RegisterExtractChangedResourcesFunc(storepb.Engine_COCKROACHDB, extractChangedResources)
}
func extractChangedResources(database string, currentSchema string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
@@ -31,13 +30,42 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
return &base.ChangeSummary{
ChangedResources: changedResources,
DMLCount: 0,
- SampleDMLS: []string{},
+ DMLStatements: []string{},
InsertCount: 0,
}, nil
}
var dmlCount, insertCount int
- var sampleDMLs []string
+ var dmlStatements []string
+ initialSearchPath := searchPath
+ // sessionSearchPath leaves out SET LOCAL, which lasts until the transaction ends, and
+ // transactionSearchPath is the session search path that ROLLBACK restores.
+ sessionSearchPath, transactionSearchPath := searchPath, searchPath
+ inTransaction := false
+ sampleDML := func(text string) {
+ dmlCount++
+ text = strings.TrimSpace(text)
+ if !strings.HasSuffix(text, ";") {
+ text += ";"
+ }
+ // The EXPLAIN connection starts with the initial search path, so a changed one is replayed.
+ if !slices.Equal(searchPath, initialSearchPath) {
+ text = base.WithSearchPath(text, searchPath)
+ }
+ dmlStatements = append(dmlStatements, text)
+ }
+ addTarget := func(rv *ast.RangeVar, affectedTable bool) {
+ db, schema, table := extractExistingRangeVarNames(rv, database, searchPath, dbMetadata)
+ changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, affectedTable)
+ }
+ // addDataModifyingCTETargets reports whether the WITH clause modifies data.
+ addDataModifyingCTETargets := func(with *ast.WithClause) bool {
+ targets := getDataModifyingCTETargets(with)
+ for _, rv := range targets {
+ addTarget(rv, false)
+ }
+ return len(targets) > 0
+ }
for _, unifiedAST := range asts {
omniAST, ok := unifiedAST.(*OmniAST)
@@ -47,11 +75,39 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
if omniAST.Node == nil {
continue
}
+ node, text := UnwrapExplainAnalyze(omniAST.Node, omniAST.Text)
+
+ switch n := node.(type) {
+ case *ast.TransactionStmt:
+ switch n.Kind {
+ case ast.TRANS_STMT_BEGIN, ast.TRANS_STMT_START:
+ // A BEGIN inside a transaction only warns.
+ if !inTransaction {
+ transactionSearchPath, inTransaction = sessionSearchPath, true
+ }
+ case ast.TRANS_STMT_COMMIT, ast.TRANS_STMT_ROLLBACK:
+ // A ROLLBACK outside a transaction only warns.
+ if n.Kind == ast.TRANS_STMT_ROLLBACK && inTransaction {
+ sessionSearchPath = transactionSearchPath
+ }
+ // COMMIT AND CHAIN and ROLLBACK AND CHAIN start the next transaction from here.
+ searchPath, transactionSearchPath, inTransaction = sessionSearchPath, sessionSearchPath, n.Chain
+ default:
+ }
+
+ case *ast.DiscardStmt:
+ if n.Target == ast.DISCARD_ALL {
+ searchPath, sessionSearchPath = initialSearchPath, initialSearchPath
+ }
- switch n := omniAST.Node.(type) {
case *ast.VariableSetStmt:
- if strings.EqualFold(n.Name, "search_path") && n.Args != nil {
- var newSearchPath []string
+ var newSearchPath []string
+ if n.Kind == ast.VAR_RESET_ALL || (strings.EqualFold(n.Name, "search_path") && (n.Kind == ast.VAR_RESET || n.Kind == ast.VAR_SET_DEFAULT)) {
+ newSearchPath = initialSearchPath
+ } else if strings.EqualFold(n.Name, "search_path") && n.Kind == ast.VAR_SET_CURRENT {
+ // SET ... FROM CURRENT makes the current search path, which a SET LOCAL may have set, the session's.
+ newSearchPath = searchPath
+ } else if strings.EqualFold(n.Name, "search_path") && n.Args != nil {
for _, arg := range n.Args.Items {
if ac, ok := arg.(*ast.A_Const); ok {
if s, ok := ac.Val.(*ast.String); ok {
@@ -59,8 +115,11 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
}
}
}
- if len(newSearchPath) > 0 {
- searchPath = newSearchPath
+ }
+ if len(newSearchPath) > 0 {
+ searchPath = newSearchPath
+ if !n.IsLocal {
+ sessionSearchPath = newSearchPath
}
}
@@ -90,12 +149,24 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
}
case *ast.RenameStmt:
- if n.Relation != nil {
+ if n.Relation == nil {
+ continue
+ }
+ // Newname names a table only when the table itself is renamed.
+ switch n.RenameType {
+ case ast.OBJECT_TABLE, ast.OBJECT_MATVIEW:
db, schema, oldTableName := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)
changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: oldTableName}, true)
if n.Newname != "" {
changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: n.Newname}, false)
}
+ case ast.OBJECT_COLUMN, ast.OBJECT_TABCONSTRAINT:
+ if n.RelationType == ast.OBJECT_TABLE || n.RelationType == ast.OBJECT_MATVIEW {
+ addTarget(n.Relation, true)
+ }
+ case ast.OBJECT_TRIGGER, ast.OBJECT_RULE, ast.OBJECT_POLICY:
+ addTarget(n.Relation, true)
+ default:
}
case *ast.IndexStmt:
@@ -105,34 +176,32 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
}
case *ast.InsertStmt:
+ hasDataModifyingCTE := addDataModifyingCTETargets(n.WithClause)
if n.Relation != nil {
- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)
- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
+ addTarget(n.Relation, false)
}
- // Count insert rows from VALUES
- if sel, ok := n.SelectStmt.(*ast.SelectStmt); ok && sel.ValuesLists != nil {
- insertCount += len(sel.ValuesLists.Items)
+ // A data-modifying CTE makes the whole statement a sample, whose estimate already
+ // includes the VALUES rows.
+ if rows, ok := getInsertValuesRowCount(n); ok && !hasDataModifyingCTE {
+ insertCount += rows
+ } else {
+ sampleDML(text)
}
case *ast.UpdateStmt:
+ addDataModifyingCTETargets(n.WithClause)
if n.Relation != nil {
- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)
- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
- }
- dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, getOmniStatementText(omniAST))
+ addTarget(n.Relation, false)
}
+ sampleDML(text)
case *ast.DeleteStmt:
+ addDataModifyingCTETargets(n.WithClause)
if n.Relation != nil {
- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)
- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
- }
- dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, getOmniStatementText(omniAST))
+ addTarget(n.Relation, false)
}
+ sampleDML(text)
+
case *ast.TruncateStmt:
if n.Relations != nil {
for _, item := range n.Relations.Items {
@@ -146,16 +215,20 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
}
case *ast.MergeStmt:
+ addDataModifyingCTETargets(n.WithClause)
if n.Relation != nil {
- db, schema, table := extractExistingRangeVarNames(n.Relation, database, searchPath, dbMetadata)
- changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
+ addTarget(n.Relation, false)
}
+ sampleDML(text)
case *ast.CreateTableAsStmt:
if n.Into != nil && n.Into.Rel != nil {
db, schema, table := extractNewRangeVarNames(n.Into.Rel, database, searchPath, dbMetadata)
changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
}
+ if query, ok := n.Query.(*ast.SelectStmt); ok && addDataModifyingCTETargets(query.WithClause) {
+ sampleDML(text)
+ }
case *ast.SelectStmt:
// SELECT ... INTO target_table is a write (classified DDL via the INTO clause,
@@ -164,6 +237,9 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
db, schema, table := extractNewRangeVarNames(into.Rel, database, searchPath, dbMetadata)
changedResources.AddTable(db, schema, &storepb.ChangedResourceTable{Name: table}, false)
}
+ if addDataModifyingCTETargets(n.WithClause) {
+ sampleDML(text)
+ }
default:
}
@@ -172,11 +248,76 @@ func extractChangedResources(database string, currentSchema string, dbMetadata *
return &base.ChangeSummary{
ChangedResources: changedResources,
DMLCount: dmlCount,
- SampleDMLS: sampleDMLs,
+ DMLStatements: dmlStatements,
InsertCount: insertCount,
}, nil
}
+// getInsertValuesRowCount returns the rows inserted by INSERT ... VALUES or INSERT ... DEFAULT
+// VALUES; ok is false when a query supplies the rows.
+func getInsertValuesRowCount(n *ast.InsertStmt) (rows int, ok bool) {
+ if n.SelectStmt == nil {
+ return 1, true
+ }
+ if sel, isSelect := n.SelectStmt.(*ast.SelectStmt); isSelect && sel.ValuesLists != nil {
+ return len(sel.ValuesLists.Items), true
+ }
+ return 0, false
+}
+
+// getWithClause returns the WITH clause of a statement, the only one where PostgreSQL allows
+// data-modifying statements.
+func getWithClause(node ast.Node) *ast.WithClause {
+ switch n := node.(type) {
+ case *ast.SelectStmt:
+ return n.WithClause
+ case *ast.InsertStmt:
+ return n.WithClause
+ case *ast.UpdateStmt:
+ return n.WithClause
+ case *ast.DeleteStmt:
+ return n.WithClause
+ case *ast.MergeStmt:
+ return n.WithClause
+ case *ast.CreateTableAsStmt:
+ return getWithClause(n.Query)
+ default:
+ return nil
+ }
+}
+
+// getDataModifyingCTETargets returns the tables that INSERT, UPDATE, DELETE, and MERGE statements in
+// a WITH clause modify. PostgreSQL only allows data-modifying statements in a top-level WITH.
+func getDataModifyingCTETargets(with *ast.WithClause) []*ast.RangeVar {
+ if with == nil || with.Ctes == nil {
+ return nil
+ }
+ var targets []*ast.RangeVar
+ for _, item := range with.Ctes.Items {
+ cte, ok := item.(*ast.CommonTableExpr)
+ if !ok {
+ continue
+ }
+ var target *ast.RangeVar
+ switch query := cte.Ctequery.(type) {
+ case *ast.InsertStmt:
+ target = query.Relation
+ case *ast.UpdateStmt:
+ target = query.Relation
+ case *ast.DeleteStmt:
+ target = query.Relation
+ case *ast.MergeStmt:
+ target = query.Relation
+ default:
+ continue
+ }
+ if target != nil {
+ targets = append(targets, target)
+ }
+ }
+ return targets
+}
+
// extractRangeVarNames extracts database, schema, table from a RangeVar with defaults.
func extractRangeVarNames(rv *ast.RangeVar, defaultDB string, searchPath []string) (string, string, string) {
db := rv.Catalogname
@@ -300,12 +441,3 @@ func extractNameListParts(nameList *ast.List, defaultDB string) (string, string,
return defaultDB, "", ""
}
}
-
-// getOmniStatementText returns the text of a statement from OmniAST, including semicolon.
-func getOmniStatementText(omniAST *OmniAST) string {
- text := strings.TrimSpace(omniAST.Text)
- if !strings.HasSuffix(text, ";") {
- text += ";"
- }
- return text
-}
diff --git a/backend/plugin/parser/pg/resource_change_test.go b/backend/plugin/parser/pg/resource_change_test.go
index 172175b4ef..4605b95c38 100644
--- a/backend/plugin/parser/pg/resource_change_test.go
+++ b/backend/plugin/parser/pg/resource_change_test.go
@@ -51,7 +51,7 @@ func TestExtractChangedResources(t *testing.T) {
want := &base.ChangeSummary{
ChangedResources: changedResources,
DMLCount: 1,
- SampleDMLS: []string{"UPDATE t1 SET c1 = 5;"},
+ DMLStatements: []string{"UPDATE t1 SET c1 = 5;"},
InsertCount: 2,
}
@@ -63,6 +63,76 @@ func TestExtractChangedResources(t *testing.T) {
require.Equal(t, want, got)
}
+func TestExtractChangedResourcesReplaysSearchPath(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ SearchPath: `"$user", public`,
+ Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}, {Name: "app"}},
+ }, nil, nil, storepb.Engine_POSTGRES, true /* caseSensitive */)
+ const statement = `UPDATE t SET c = 1;
+SET search_path TO app;
+UPDATE t SET c = 2;
+SET search_path TO public;
+UPDATE t SET c = 3;
+SET search_path TO app;
+RESET search_path;
+UPDATE t SET c = 4;
+SET search_path TO app;
+SET search_path TO DEFAULT;
+UPDATE t SET c = 5;
+BEGIN;
+SET LOCAL search_path TO app;
+UPDATE t SET c = 6;
+COMMIT;
+UPDATE t SET c = 7;
+BEGIN;
+SET search_path TO app;
+ROLLBACK;
+UPDATE t SET c = 8;
+BEGIN;
+SET search_path TO app;
+COMMIT AND CHAIN;
+ROLLBACK;
+UPDATE t SET c = 9;
+DISCARD ALL;
+UPDATE t SET c = 10;
+SET search_path TO app;
+BEGIN;
+SET search_path TO other;
+BEGIN;
+SET search_path TO third;
+ROLLBACK;
+UPDATE t SET c = 11;
+SET search_path TO other;
+SET LOCAL search_path TO app;
+ROLLBACK;
+UPDATE t SET c = 12;
+BEGIN;
+SET LOCAL search_path TO app;
+SET search_path FROM CURRENT;
+COMMIT;
+UPDATE t SET c = 13;`
+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)
+ require.NoError(t, err)
+ got, err := extractChangedResources("db", "", dbMetadata, base.ExtractASTs(stmts), statement)
+ require.NoError(t, err)
+ require.Equal(t, []string{
+ "UPDATE t SET c = 1;",
+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 2;",
+ "UPDATE t SET c = 3;",
+ "UPDATE t SET c = 4;",
+ "UPDATE t SET c = 5;",
+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 6;",
+ "UPDATE t SET c = 7;",
+ "UPDATE t SET c = 8;",
+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 9;",
+ "UPDATE t SET c = 10;",
+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 11;",
+ "SET LOCAL search_path TO \"other\";\nUPDATE t SET c = 12;",
+ "SET LOCAL search_path TO \"app\";\nUPDATE t SET c = 13;",
+ }, got.DMLStatements)
+}
+
func TestExtractChangedResourcesSelectedSchemaFallsBackToPublicForExistingTarget(t *testing.T) {
const statement = `INSERT INTO customer VALUES (1);`
@@ -159,6 +229,182 @@ func TestExtractChangedResourcesMerge(t *testing.T) {
require.Equal(t, want, got.ChangedResources)
}
+func TestExtractChangedResourcesDMLCounts(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)
+ for _, tc := range []struct {
+ name string
+ statement string
+ wantDMLCount int
+ wantInsertCount int
+ wantTables []string
+ }{
+ {
+ name: "insert values",
+ statement: `INSERT INTO t VALUES (1), (2), (3);`,
+ wantInsertCount: 3,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert values on conflict",
+ statement: `INSERT INTO t VALUES (1), (2) ON CONFLICT (id) DO UPDATE SET c = 1;`,
+ wantInsertCount: 2,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert default values",
+ statement: `INSERT INTO t DEFAULT VALUES;`,
+ wantInsertCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert select",
+ statement: `INSERT INTO t SELECT * FROM t2 ON CONFLICT DO NOTHING;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "insert select with read-only CTE",
+ statement: `WITH x AS (SELECT id FROM t2) INSERT INTO t SELECT id FROM x;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "merge",
+ statement: `MERGE INTO t USING t2 ON t.id = t2.id WHEN MATCHED THEN UPDATE SET c = t2.c;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "data-modifying CTE feeding insert",
+ statement: `WITH d AS (DELETE FROM t WHERE id > 0 RETURNING *) INSERT INTO t2 SELECT * FROM d;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "data-modifying CTE feeding insert values",
+ statement: `WITH i AS (INSERT INTO t VALUES (1) RETURNING id) INSERT INTO t2 VALUES (2), (3);`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2"},
+ },
+ {
+ name: "data-modifying CTE in select",
+ statement: `WITH d AS (DELETE FROM t RETURNING 1) SELECT count(*) FROM d;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t"},
+ },
+ {
+ name: "data-modifying CTEs feeding delete",
+ statement: `WITH u AS (UPDATE t SET c = 1 RETURNING id), m AS (MERGE INTO t3 USING t2 ON t3.id = t2.id WHEN MATCHED THEN DELETE) DELETE FROM t2 WHERE id IN (SELECT id FROM u);`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t2", "t3"},
+ },
+ {
+ name: "data-modifying CTE in create table as",
+ statement: `CREATE TABLE t_moved AS WITH d AS (DELETE FROM t RETURNING *) SELECT * FROM d;`,
+ wantDMLCount: 1,
+ wantTables: []string{"t", "t_moved"},
+ },
+ {
+ name: "read-only CTE in select",
+ statement: `WITH x AS (SELECT id FROM t) SELECT * FROM x;`,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ want := model.NewChangedResources(dbMetadata)
+ for _, table := range tc.wantTables {
+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ var wantSamples []string
+ if tc.wantDMLCount > 0 {
+ wantSamples = []string{tc.statement}
+ }
+
+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, tc.statement)
+ require.NoError(t, err)
+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, &base.ChangeSummary{
+ ChangedResources: want,
+ DMLCount: tc.wantDMLCount,
+ DMLStatements: wantSamples,
+ InsertCount: tc.wantInsertCount,
+ }, got)
+ })
+ }
+}
+
+func TestExtractChangedResourcesExplainAnalyze(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)
+ const statement = `EXPLAIN (ANALYZE, BUFFERS) WITH d AS (DELETE FROM t RETURNING id) INSERT INTO t2 SELECT id FROM d;
+EXPLAIN ANALYZE MERGE INTO t3 USING t ON t3.id = t.id WHEN MATCHED THEN DELETE;
+EXPLAIN UPDATE t4 SET c = 1;
+EXPLAIN (ANALYZE FALSE) DELETE FROM t5;
+EXPLAIN ANALYZE WITH d AS (DELETE FROM t6 RETURNING id) SELECT count(*) FROM d ORDER BY 1;`
+
+ want := model.NewChangedResources(dbMetadata)
+ for _, table := range []string{"t", "t2", "t3", "t6"} {
+ want.AddTable("db", "public", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, statement)
+ require.NoError(t, err)
+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), statement)
+ require.NoError(t, err)
+ require.Equal(t, &base.ChangeSummary{
+ ChangedResources: want,
+ DMLCount: 3,
+ DMLStatements: []string{
+ "WITH d AS (DELETE FROM t RETURNING id) INSERT INTO t2 SELECT id FROM d;",
+ "MERGE INTO t3 USING t ON t3.id = t.id WHEN MATCHED THEN DELETE;",
+ "WITH d AS (DELETE FROM t6 RETURNING id) SELECT count(*) FROM d ORDER BY 1;",
+ },
+ }, got)
+}
+
+func TestExtractChangedResourcesRename(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {Name: "t", RowCount: 10},
+ {Name: "t2", RowCount: 1000},
+ {Name: "t3", RowCount: 1_000_000},
+ },
+ }},
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)
+ for _, tc := range []struct {
+ statement string
+ wantTables []string
+ wantAffectedRows int64
+ }{
+ {statement: `ALTER TABLE t RENAME TO t_new;`, wantTables: []string{"t", "t_new"}, wantAffectedRows: 10},
+ {statement: `ALTER TABLE t2 RENAME COLUMN c TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER TABLE t2 RENAME c TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER TABLE t2 RENAME CONSTRAINT t2_pkey TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER TRIGGER trg ON t2 RENAME TO t3;`, wantTables: []string{"t2"}, wantAffectedRows: 1000},
+ {statement: `ALTER INDEX t2_idx RENAME TO t3;`},
+ {statement: `ALTER VIEW v RENAME COLUMN c TO t3;`},
+ } {
+ t.Run(tc.statement, func(t *testing.T) {
+ stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, tc.statement)
+ require.NoError(t, err)
+ got, err := extractChangedResources("db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)
+ require.NoError(t, err)
+
+ var gotTables []string
+ for _, database := range got.ChangedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ gotTables = append(gotTables, table.GetName())
+ }
+ }
+ }
+ require.Equal(t, tc.wantTables, gotTables)
+ require.Equal(t, tc.wantAffectedRows, got.ChangedResources.CountAffectedTableRows())
+ })
+ }
+}
+
func TestExtractChangedResourcesCTAS(t *testing.T) {
dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_POSTGRES, true /* caseSensitive */)
for _, tc := range []struct {
diff --git a/backend/plugin/parser/pg/statement_type.go b/backend/plugin/parser/pg/statement_type.go
index 148870e55e..85957bbf10 100644
--- a/backend/plugin/parser/pg/statement_type.go
+++ b/backend/plugin/parser/pg/statement_type.go
@@ -34,8 +34,8 @@ func GetStatementTypes(asts []base.AST) ([]StatementTypeWithPosition, error) {
continue
}
- stmtType := classifyStatementType(omniAST.Node)
- if stmtType == storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED {
+ stmtTypes := classifyStatementTypes(omniAST.Node)
+ if len(stmtTypes) == 0 {
continue
}
@@ -50,11 +50,13 @@ func GetStatementTypes(asts []base.AST) ([]StatementTypeWithPosition, error) {
endLine += strings.Count(omniAST.Text, "\n")
}
- allResults = append(allResults, StatementTypeWithPosition{
- Type: stmtType,
- Line: endLine,
- Text: omniAST.Text,
- })
+ for _, stmtType := range stmtTypes {
+ allResults = append(allResults, StatementTypeWithPosition{
+ Type: stmtType,
+ Line: endLine,
+ Text: omniAST.Text,
+ })
+ }
}
return allResults, nil
diff --git a/backend/plugin/parser/pg/statement_type_antlr.go b/backend/plugin/parser/pg/statement_type_antlr.go
index 0430f12df9..1a50b7349d 100644
--- a/backend/plugin/parser/pg/statement_type_antlr.go
+++ b/backend/plugin/parser/pg/statement_type_antlr.go
@@ -1,11 +1,34 @@
package pg
import (
+ "slices"
+
"github.com/bytebase/omni/pg/ast"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
+// classifyStatementTypes returns the type of the statement followed by the types of its
+// data-modifying CTEs, each type once.
+func classifyStatementTypes(node ast.Node) []storepb.StatementType {
+ node, _ = UnwrapExplainAnalyze(node, "")
+ var types []storepb.StatementType
+ add := func(statementType storepb.StatementType) {
+ if statementType != storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED && !slices.Contains(types, statementType) {
+ types = append(types, statementType)
+ }
+ }
+ add(classifyStatementType(node))
+ if with := getWithClause(node); with != nil && with.Ctes != nil {
+ for _, item := range with.Ctes.Items {
+ if cte, ok := item.(*ast.CommonTableExpr); ok {
+ add(classifyStatementType(cte.Ctequery))
+ }
+ }
+ }
+ return types
+}
+
// classifyStatementType returns the statement type for an omni AST node.
func classifyStatementType(node ast.Node) storepb.StatementType {
switch n := node.(type) {
@@ -35,6 +58,16 @@ func classifyStatementType(node ast.Node) storepb.StatementType {
return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
case *ast.CreateEnumStmt:
return storepb.StatementType_CREATE_TYPE
+ case *ast.CreateTableAsStmt:
+ if n.Objtype == ast.OBJECT_MATVIEW {
+ return storepb.StatementType_CREATE_VIEW
+ }
+ return storepb.StatementType_CREATE_TABLE
+ case *ast.SelectStmt:
+ if hasOmniIntoClause(n) {
+ return storepb.StatementType_CREATE_TABLE
+ }
+ return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
// DDL - DROP
case *ast.DropStmt:
@@ -75,6 +108,8 @@ func classifyStatementType(node ast.Node) storepb.StatementType {
return storepb.StatementType_UPDATE
case *ast.DeleteStmt:
return storepb.StatementType_DELETE
+ case *ast.MergeStmt:
+ return storepb.StatementType_MERGE
default:
return storepb.StatementType_STATEMENT_TYPE_UNSPECIFIED
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.json b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.json
new file mode 100644
index 0000000000..0a3ad18552
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_insert.json
@@ -0,0 +1,59 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Insert",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big_archive",
+ "Alias": "big_archive",
+ "Startup Cost": 180.00,
+ "Total Cost": 280.00,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "ModifyTable",
+ "Operation": "Delete",
+ "Parent Relationship": "InitPlan",
+ "Subplan Name": "CTE moved",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 5000,
+ "Plan Width": 6,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 5000,
+ "Plan Width": 6,
+ "Filter": "(s_id <= 50)"
+ }
+ ]
+ },
+ {
+ "Node Type": "CTE Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "CTE Name": "moved",
+ "Alias": "moved",
+ "Startup Cost": 0.00,
+ "Total Cost": 100.00,
+ "Plan Rows": 5000,
+ "Plan Width": 12
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.json b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.json
new file mode 100644
index 0000000000..840159df88
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/cte_delete_select.json
@@ -0,0 +1,58 @@
+[
+ {
+ "Plan": {
+ "Node Type": "Aggregate",
+ "Strategy": "Plain",
+ "Partial Mode": "Simple",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 292.50,
+ "Total Cost": 292.51,
+ "Plan Rows": 1,
+ "Plan Width": 8,
+ "Plans": [
+ {
+ "Node Type": "ModifyTable",
+ "Operation": "Delete",
+ "Parent Relationship": "InitPlan",
+ "Subplan Name": "CTE d",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 5000,
+ "Plan Width": 6,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 5000,
+ "Plan Width": 6,
+ "Filter": "(s_id <= 50)"
+ }
+ ]
+ },
+ {
+ "Node Type": "CTE Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "CTE Name": "d",
+ "Alias": "d",
+ "Startup Cost": 0.00,
+ "Total Cost": 100.00,
+ "Plan Rows": 5000,
+ "Plan Width": 0
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.json
new file mode 100644
index 0000000000..9510c2dd5f
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_in_subquery.json
@@ -0,0 +1,70 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Delete",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Hash Join",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Join Type": "Inner",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 1000,
+ "Plan Width": 12,
+ "Inner Unique": true,
+ "Hash Cond": "(big.s_id = s.id)",
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 155.00,
+ "Plan Rows": 10000,
+ "Plan Width": 10
+ },
+ {
+ "Node Type": "Hash",
+ "Parent Relationship": "Inner",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Filter": "flag"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.json
new file mode 100644
index 0000000000..ba31ecf827
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_inheritance_parent.json
@@ -0,0 +1,67 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Delete",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "inh_parent",
+ "Alias": "inh_parent",
+ "Startup Cost": 0.00,
+ "Total Cost": 18.00,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Target Tables": [
+ {
+ "Relation Name": "inh_parent",
+ "Alias": "inh_parent_1"
+ },
+ {
+ "Relation Name": "inh_child",
+ "Alias": "inh_parent_2"
+ }
+ ],
+ "Plans": [
+ {
+ "Node Type": "Append",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 0.00,
+ "Total Cost": 18.00,
+ "Plan Rows": 101,
+ "Plan Width": 10,
+ "Subplans Removed": 0,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Member",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "inh_parent",
+ "Alias": "inh_parent_1",
+ "Startup Cost": 0.00,
+ "Total Cost": 0.00,
+ "Plan Rows": 1,
+ "Plan Width": 10,
+ "Filter": "(c < 10)"
+ },
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Member",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "inh_child",
+ "Alias": "inh_parent_2",
+ "Startup Cost": 0.00,
+ "Total Cost": 17.50,
+ "Plan Rows": 100,
+ "Plan Width": 10,
+ "Filter": "(c < 10)"
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.json b/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.json
new file mode 100644
index 0000000000..9510c2dd5f
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/delete_using_join.json
@@ -0,0 +1,70 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Delete",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Hash Join",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Join Type": "Inner",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 1000,
+ "Plan Width": 12,
+ "Inner Unique": true,
+ "Hash Cond": "(big.s_id = s.id)",
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 155.00,
+ "Plan Rows": 10000,
+ "Plan Width": 10
+ },
+ {
+ "Node Type": "Hash",
+ "Parent Relationship": "Inner",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Filter": "flag"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_select.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_select.json
new file mode 100644
index 0000000000..e1e145deef
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_select.json
@@ -0,0 +1,45 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Insert",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t2",
+ "Alias": "t2",
+ "Startup Cost": 5.05,
+ "Total Cost": 13.30,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Bitmap Heap Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 13.30,
+ "Plan Rows": 100,
+ "Plan Width": 18,
+ "Recheck Cond": "(c < 10)",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Index Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Index Name": "t_c_idx",
+ "Startup Cost": 0.00,
+ "Total Cost": 5.03,
+ "Plan Rows": 100,
+ "Plan Width": 0,
+ "Index Cond": "(c < 10)"
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.json
new file mode 100644
index 0000000000..ab19a70983
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_select_on_conflict_do_nothing.json
@@ -0,0 +1,32 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Insert",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big_pk",
+ "Alias": "big_pk",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Conflict Resolution": "NOTHING",
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 1000,
+ "Plan Width": 12,
+ "Filter": "(s_id <= 10)"
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.json b/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.json
new file mode 100644
index 0000000000..bb259bc7d1
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/insert_values_on_conflict.json
@@ -0,0 +1,31 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Insert",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big_pk",
+ "Alias": "big_pk",
+ "Startup Cost": 0.00,
+ "Total Cost": 0.03,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Conflict Resolution": "UPDATE",
+ "Conflict Arbiter Indexes": ["big_pk_pkey"],
+ "Plans": [
+ {
+ "Node Type": "Values Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Alias": "*VALUES*",
+ "Startup Cost": 0.00,
+ "Total Cost": 0.03,
+ "Plan Rows": 2,
+ "Plan Width": 12
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/merge.json b/backend/plugin/parser/pg/test-data/explain-plan/merge.json
new file mode 100644
index 0000000000..7fadbf86d7
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/merge.json
@@ -0,0 +1,69 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Merge",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 3.25,
+ "Total Cost": 185.61,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Hash Join",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Join Type": "Inner",
+ "Startup Cost": 3.25,
+ "Total Cost": 185.61,
+ "Plan Rows": 10000,
+ "Plan Width": 12,
+ "Inner Unique": true,
+ "Hash Cond": "(big.s_id = s.id)",
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 155.00,
+ "Plan Rows": 10000,
+ "Plan Width": 10
+ },
+ {
+ "Node Type": "Hash",
+ "Parent Relationship": "Inner",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 100,
+ "Plan Width": 10,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 100,
+ "Plan Width": 10
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/select.json b/backend/plugin/parser/pg/test-data/explain-plan/select.json
new file mode 100644
index 0000000000..3faddc7975
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/select.json
@@ -0,0 +1,30 @@
+[
+ {
+ "Plan": {
+ "Node Type": "Bitmap Heap Scan",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 13.30,
+ "Plan Rows": 100,
+ "Plan Width": 18,
+ "Recheck Cond": "(c < 10)",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Index Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Index Name": "t_c_idx",
+ "Startup Cost": 0.00,
+ "Total Cost": 5.03,
+ "Plan Rows": 100,
+ "Plan Width": 0,
+ "Index Cond": "(c < 10)"
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update.json b/backend/plugin/parser/pg/test-data/explain-plan/update.json
new file mode 100644
index 0000000000..58e0a48df0
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update.json
@@ -0,0 +1,45 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 13.30,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Bitmap Heap Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 13.30,
+ "Plan Rows": 100,
+ "Plan Width": 38,
+ "Recheck Cond": "(c < 10)",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Index Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Index Name": "t_c_idx",
+ "Startup Cost": 0.00,
+ "Total Cost": 5.03,
+ "Plan Rows": 100,
+ "Plan Width": 0,
+ "Index Cond": "(c < 10)"
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.json b/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.json
new file mode 100644
index 0000000000..0eab80be43
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_correlated_set_subquery.json
@@ -0,0 +1,84 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 239.80,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Bitmap Heap Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Startup Cost": 5.05,
+ "Total Cost": 239.80,
+ "Plan Rows": 100,
+ "Plan Width": 38,
+ "Recheck Cond": "(c < 10)",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Index Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Index Name": "t_c_idx",
+ "Startup Cost": 0.00,
+ "Total Cost": 5.03,
+ "Plan Rows": 100,
+ "Plan Width": 0,
+ "Index Cond": "(c < 10)"
+ },
+ {
+ "Node Type": "Result",
+ "Parent Relationship": "SubPlan",
+ "Subplan Name": "SubPlan 2",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.25,
+ "Total Cost": 2.26,
+ "Plan Rows": 1,
+ "Plan Width": 32,
+ "Plans": [
+ {
+ "Node Type": "Limit",
+ "Parent Relationship": "InitPlan",
+ "Subplan Name": "InitPlan 1",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 0.00,
+ "Total Cost": 2.25,
+ "Plan Rows": 1,
+ "Plan Width": 4,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.25,
+ "Plan Rows": 1,
+ "Plan Width": 4,
+ "Filter": "(id = t.grp)"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.json b/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.json
new file mode 100644
index 0000000000..3a6d75a273
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_costs_off.json
@@ -0,0 +1,33 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Heap Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "t",
+ "Alias": "t",
+ "Recheck Cond": "(c < 10)",
+ "Plans": [
+ {
+ "Node Type": "Bitmap Index Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Index Name": "t_c_idx",
+ "Index Cond": "(c < 10)"
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.json b/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.json
new file mode 100644
index 0000000000..988b5c8636
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_from_join.json
@@ -0,0 +1,70 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Hash Join",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Join Type": "Inner",
+ "Startup Cost": 2.12,
+ "Total Cost": 184.49,
+ "Plan Rows": 1000,
+ "Plan Width": 16,
+ "Inner Unique": true,
+ "Hash Cond": "(big.s_id = s.id)",
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 155.00,
+ "Plan Rows": 10000,
+ "Plan Width": 10
+ },
+ {
+ "Node Type": "Hash",
+ "Parent Relationship": "Inner",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 10,
+ "Filter": "flag"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.json b/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.json
new file mode 100644
index 0000000000..997434ccb7
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_initplan.json
@@ -0,0 +1,59 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 2.03,
+ "Total Cost": 182.03,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Plans": [
+ {
+ "Node Type": "Aggregate",
+ "Strategy": "Plain",
+ "Partial Mode": "Simple",
+ "Parent Relationship": "InitPlan",
+ "Subplan Name": "InitPlan 1",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 2.02,
+ "Total Cost": 2.03,
+ "Plan Rows": 1,
+ "Plan Width": 8,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "s",
+ "Alias": "s",
+ "Startup Cost": 0.00,
+ "Total Cost": 2.00,
+ "Plan Rows": 10,
+ "Plan Width": 0,
+ "Filter": "flag"
+ }
+ ]
+ },
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "big",
+ "Alias": "big",
+ "Startup Cost": 0.00,
+ "Total Cost": 180.00,
+ "Plan Rows": 3333,
+ "Plan Width": 10,
+ "Filter": "(s_id <= (InitPlan 1).col1)"
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.json b/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.json
new file mode 100644
index 0000000000..33b92b6713
--- /dev/null
+++ b/backend/plugin/parser/pg/test-data/explain-plan/update_partitioned.json
@@ -0,0 +1,67 @@
+[
+ {
+ "Plan": {
+ "Node Type": "ModifyTable",
+ "Operation": "Update",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "parted",
+ "Alias": "parted",
+ "Startup Cost": 0.00,
+ "Total Cost": 19.00,
+ "Plan Rows": 0,
+ "Plan Width": 0,
+ "Target Tables": [
+ {
+ "Relation Name": "parted_p1",
+ "Alias": "parted_1"
+ },
+ {
+ "Relation Name": "parted_p2",
+ "Alias": "parted_2"
+ }
+ ],
+ "Plans": [
+ {
+ "Node Type": "Append",
+ "Parent Relationship": "Outer",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Startup Cost": 0.00,
+ "Total Cost": 19.00,
+ "Plan Rows": 100,
+ "Plan Width": 14,
+ "Subplans Removed": 0,
+ "Plans": [
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Member",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "parted_p1",
+ "Alias": "parted_1",
+ "Startup Cost": 0.00,
+ "Total Cost": 9.25,
+ "Plan Rows": 50,
+ "Plan Width": 14,
+ "Filter": "(c < 10)"
+ },
+ {
+ "Node Type": "Seq Scan",
+ "Parent Relationship": "Member",
+ "Parallel Aware": false,
+ "Async Capable": false,
+ "Relation Name": "parted_p2",
+ "Alias": "parted_2",
+ "Startup Cost": 0.00,
+ "Total Cost": 9.25,
+ "Plan Rows": 50,
+ "Plan Width": 14,
+ "Filter": "(c < 10)"
+ }
+ ]
+ }
+ ]
+ }
+ }
+]
diff --git a/backend/plugin/parser/pg/test-data/query_type.yaml b/backend/plugin/parser/pg/test-data/query_type.yaml
index 26859cd000..4b3639a899 100644
--- a/backend/plugin/parser/pg/test-data/query_type.yaml
+++ b/backend/plugin/parser/pg/test-data/query_type.yaml
@@ -590,3 +590,114 @@
results: []
sourcecolumns: []
predicatecolumns: []
+- description: SELECT with a data-modifying CTE is DML
+ statement: WITH d AS (DELETE FROM t1 WHERE a = '1' RETURNING a) SELECT count(*) FROM d;
+ defaultDatabase: db
+ metadata: |-
+ {
+ "name": "db",
+ "schemas": [
+ {
+ "name": "public",
+ "tables": [
+ {
+ "name": "t1",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ },
+ {
+ "name": "t2",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ querySpan:
+ type: 5
+ results: []
+ sourcecolumns: []
+ predicatecolumns: []
+- description: EXPLAIN ANALYZE executes MERGE
+ statement: EXPLAIN ANALYZE MERGE INTO t1 USING t2 ON t1.a = t2.a WHEN NOT MATCHED THEN INSERT (a) VALUES (t2.a);
+ defaultDatabase: db
+ metadata: |-
+ {
+ "name": "db",
+ "schemas": [
+ {
+ "name": "public",
+ "tables": [
+ {
+ "name": "t1",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ },
+ {
+ "name": "t2",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ querySpan:
+ type: 5
+ results: []
+ sourcecolumns: []
+ predicatecolumns: []
+- description: EXPLAIN ANALYZE executes a data-modifying CTE
+ statement: EXPLAIN ANALYZE WITH i AS (INSERT INTO t1 (a) VALUES ('1') RETURNING a) SELECT count(*) FROM i;
+ defaultDatabase: db
+ metadata: |-
+ {
+ "name": "db",
+ "schemas": [
+ {
+ "name": "public",
+ "tables": [
+ {
+ "name": "t1",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ },
+ {
+ "name": "t2",
+ "columns": [
+ {
+ "name": "a",
+ "type": "text"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ querySpan:
+ type: 5
+ results: []
+ sourcecolumns: []
+ predicatecolumns: []
diff --git a/backend/plugin/parser/pg/test-data/test_statement_type.yaml b/backend/plugin/parser/pg/test-data/test_statement_type.yaml
index 17a05c94f8..712d2cd200 100644
--- a/backend/plugin/parser/pg/test-data/test_statement_type.yaml
+++ b/backend/plugin/parser/pg/test-data/test_statement_type.yaml
@@ -129,3 +129,42 @@
- statement: DROP MATERIALIZED VIEW mv1;
want:
- DROP_TABLE
+- statement: MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;
+ want:
+ - MERGE
+- statement: WITH d AS (DELETE FROM t1 RETURNING id) SELECT count(*) FROM d;
+ want:
+ - DELETE
+- statement: WITH d AS (DELETE FROM t1 RETURNING *), u AS (UPDATE t3 SET c = 1 RETURNING id) INSERT INTO t2 SELECT * FROM d;
+ want:
+ - INSERT
+ - DELETE
+ - UPDATE
+- statement: EXPLAIN ANALYZE MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;
+ want:
+ - MERGE
+- statement: EXPLAIN (ANALYZE, BUFFERS) WITH d AS (DELETE FROM t1 RETURNING id) UPDATE t2 SET c = 1 FROM d;
+ want:
+ - UPDATE
+ - DELETE
+- statement: EXPLAIN WITH d AS (DELETE FROM t1 RETURNING id) SELECT count(*) FROM d;
+ want: []
+- statement: EXPLAIN (ANALYZE FALSE) DELETE FROM t1;
+ want: []
+- statement: EXPLAIN (ANALYZE 1) DELETE FROM t1;
+ want:
+ - DELETE
+- statement: CREATE TABLE archive AS WITH d AS (DELETE FROM t1 RETURNING *) SELECT * FROM d;
+ want:
+ - CREATE_TABLE
+ - DELETE
+- statement: WITH d AS (DELETE FROM t1 RETURNING *) SELECT * INTO archive FROM d;
+ want:
+ - CREATE_TABLE
+ - DELETE
+- statement: CREATE MATERIALIZED VIEW mv AS SELECT * FROM t1;
+ want:
+ - CREATE_VIEW
+- statement: CREATE FUNCTION purge() RETURNS bigint LANGUAGE sql BEGIN ATOMIC WITH d AS (DELETE FROM t1 RETURNING 1) SELECT count(*) FROM d; END;
+ want:
+ - CREATE_FUNCTION
diff --git a/backend/plugin/parser/plsql/resource_change.go b/backend/plugin/parser/plsql/resource_change.go
index ed0efc40c0..d9d16ae0fd 100644
--- a/backend/plugin/parser/plsql/resource_change.go
+++ b/backend/plugin/parser/plsql/resource_change.go
@@ -5,7 +5,6 @@ import (
oracleast "github.com/bytebase/omni/oracle/ast"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -35,7 +34,7 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
return &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: extractor.sampleDMLs,
+ DMLStatements: extractor.dmlStatements,
DMLCount: extractor.dmlCount,
InsertCount: extractor.insertCount,
}, nil
@@ -46,7 +45,7 @@ type omniChangedResourceExtractor struct {
dbMetadata *model.DatabaseMetadata
changedResources *model.ChangedResources
statement string
- sampleDMLs []string
+ dmlStatements []string
dmlCount int
insertCount int
}
@@ -215,9 +214,7 @@ func (e *omniChangedResourceExtractor) addIndexTable(name *oracleast.ObjectName)
func (e *omniChangedResourceExtractor) trackDML(text string) {
e.dmlCount++
- if len(e.sampleDMLs) < common.MaximumLintExplainSize {
- e.sampleDMLs = append(e.sampleDMLs, omniStatementText(text))
- }
+ e.dmlStatements = append(e.dmlStatements, omniStatementText(text))
}
func omniStatementText(text string) string {
diff --git a/backend/plugin/parser/plsql/resource_change_test.go b/backend/plugin/parser/plsql/resource_change_test.go
index f88faf6b0c..8abca22d86 100644
--- a/backend/plugin/parser/plsql/resource_change_test.go
+++ b/backend/plugin/parser/plsql/resource_change_test.go
@@ -84,7 +84,7 @@ DELETE FROM t2 WHERE c1 = 1;`
changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T2"}, false)
want := &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: []string{
+ DMLStatements: []string{
"INSERT INTO t1 SELECT * FROM t2;",
"UPDATE t1 SET c1 = 5 WHERE c2 = 1;",
"DELETE FROM t2 WHERE c1 = 1;",
@@ -153,7 +153,7 @@ END;`
changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)
want := &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: []string{
+ DMLStatements: []string{
"UPDATE t1 SET c1 = 5;",
},
DMLCount: 1,
diff --git a/backend/plugin/parser/redshift/resource_change.go b/backend/plugin/parser/redshift/resource_change.go
index ce4babbf86..1c3c04968a 100644
--- a/backend/plugin/parser/redshift/resource_change.go
+++ b/backend/plugin/parser/redshift/resource_change.go
@@ -6,7 +6,6 @@ import (
omniredshift "github.com/bytebase/omni/redshift"
redshiftast "github.com/bytebase/omni/redshift/ast"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -17,52 +16,161 @@ func init() {
}
func extractChangedResources(database string, schema string, dbMetadata *model.DatabaseMetadata, _ []base.AST, statement string) (*base.ChangeSummary, error) {
+ if schema == "" {
+ schema = "public"
+ }
omniSummary, err := omniredshift.ExtractChangedResources(statement, database, schema)
if err != nil {
return nil, err
}
+ stmts, err := ParseRedshift(statement)
+ if err != nil {
+ return nil, err
+ }
+ ddlAffected, err := ddlAffectedTables(stmts, database, schema)
+ if err != nil {
+ return nil, err
+ }
- changedResources := model.NewChangedResources(dbMetadata)
+ summary := &base.ChangeSummary{ChangedResources: model.NewChangedResources(dbMetadata)}
for _, table := range omniSummary.Tables {
- changedResources.AddTable(
+ summary.ChangedResources.AddTable(
table.Database,
table.Schema,
- &storepb.ChangedResourceTable{
- Name: table.Name,
- },
- table.Affected,
+ &storepb.ChangedResourceTable{Name: table.Name},
+ ddlAffected[tableKey{database: table.Database, schema: table.Schema, name: table.Name}],
)
}
- sampleDMLs, err := extractOmniSampleDMLs(statement)
- if err != nil {
- return nil, err
+ searchPath := schema
+ // replaySearchPath is the search path set after the starting one, which the EXPLAIN connection
+ // does not have.
+ var replaySearchPath []string
+ sampleDML := func(text string) {
+ summary.DMLCount++
+ text = strings.TrimSpace(text)
+ if len(replaySearchPath) > 0 {
+ text = base.WithSearchPath(text, replaySearchPath)
+ }
+ summary.DMLStatements = append(summary.DMLStatements, text)
+ }
+ addTable := func(rv *redshiftast.RangeVar, affectedTable bool) {
+ tableDatabase, tableSchema := rv.Catalogname, rv.Schemaname
+ if tableDatabase == "" {
+ tableDatabase = database
+ }
+ if tableSchema == "" {
+ tableSchema = searchPath
+ }
+ summary.ChangedResources.AddTable(tableDatabase, tableSchema, &storepb.ChangedResourceTable{Name: rv.Relname}, affectedTable)
+ }
+ for _, stmt := range stmts {
+ switch n := stmt.AST.(type) {
+ case *redshiftast.InsertStmt:
+ if rows, ok := insertValuesRowCount(n); ok {
+ summary.InsertCount += rows
+ } else {
+ sampleDML(stmt.Text)
+ }
+ case *redshiftast.UpdateStmt, *redshiftast.DeleteStmt:
+ sampleDML(stmt.Text)
+ case *redshiftast.MergeStmt:
+ // Redshift documents EXPLAIN only for SELECT, CREATE TABLE AS, INSERT, UPDATE, and DELETE.
+ summary.DMLCount++
+ // omni records no MERGE target.
+ if n.Relation != nil {
+ addTable(n.Relation, false)
+ }
+ case *redshiftast.TruncateStmt:
+ if n.Relations == nil {
+ continue
+ }
+ for _, item := range n.Relations.Items {
+ if rv, ok := item.(*redshiftast.RangeVar); ok {
+ addTable(rv, true)
+ }
+ }
+ case *redshiftast.VariableSetStmt:
+ if schemas, ok := searchPathFromSet(n); ok {
+ searchPath, replaySearchPath = schema, schemas
+ if len(schemas) > 0 {
+ searchPath = schemas[0]
+ }
+ }
+ default:
+ }
}
+ return summary, nil
+}
- return &base.ChangeSummary{
- ChangedResources: changedResources,
- SampleDMLS: sampleDMLs,
- DMLCount: omniSummary.DMLCount,
- InsertCount: omniSummary.InsertCount,
- }, nil
+type tableKey struct {
+ database string
+ schema string
+ name string
}
-func extractOmniSampleDMLs(statement string) ([]string, error) {
- stmts, err := ParseRedshift(statement)
+// ddlAffectedTables returns the tables whose existing rows the script's DDL affects.
+// omni also flags DML targets and merges flags across statements, so the script it
+// reads here leaves out the DML.
+func ddlAffectedTables(stmts []omniredshift.Statement, database, schema string) (map[tableKey]bool, error) {
+ var texts []string
+ for _, stmt := range stmts {
+ switch stmt.AST.(type) {
+ case *redshiftast.InsertStmt, *redshiftast.UpdateStmt, *redshiftast.DeleteStmt, *redshiftast.MergeStmt:
+ default:
+ texts = append(texts, stmt.Text)
+ }
+ }
+ // A statement's text lacks its semicolon when a comment precedes the semicolon.
+ summary, err := omniredshift.ExtractChangedResources(strings.Join(texts, ";\n"), database, schema)
if err != nil {
return nil, err
}
+ affected := make(map[tableKey]bool)
+ for _, table := range summary.Tables {
+ if table.Affected {
+ affected[tableKey{database: table.Database, schema: table.Schema, name: table.Name}] = true
+ }
+ }
+ return affected, nil
+}
- var sampleDMLs []string
- for _, stmt := range stmts {
- if len(sampleDMLs) >= common.MaximumLintExplainSize {
- break
+// insertValuesRowCount returns how many rows INSERT ... VALUES or INSERT ... DEFAULT
+// VALUES adds, and false for an INSERT whose rows come from a query.
+func insertValuesRowCount(n *redshiftast.InsertStmt) (int, bool) {
+ if n.SelectStmt == nil {
+ return 1, true
+ }
+ if s, ok := n.SelectStmt.(*redshiftast.SelectStmt); ok && s.ValuesLists != nil {
+ return len(s.ValuesLists.Items), true
+ }
+ return 0, false
+}
+
+// searchPathFromSet returns the schemas a SET search_path names, skipping "$user", or none for RESET
+// or SET ... TO DEFAULT, which restore the starting search path. ok is false for any other statement.
+// Like omni's ExtractChangedResources, unqualified names resolve to the first schema.
+func searchPathFromSet(n *redshiftast.VariableSetStmt) ([]string, bool) {
+ if !strings.EqualFold(n.Name, "search_path") {
+ return nil, false
+ }
+ if n.Kind == redshiftast.VAR_SET_DEFAULT || n.Kind == redshiftast.VAR_RESET {
+ return nil, true
+ }
+ if n.Args == nil {
+ return nil, false
+ }
+ var schemas []string
+ for _, arg := range n.Args.Items {
+ c, ok := arg.(*redshiftast.A_Const)
+ if !ok {
+ continue
}
- switch stmt.AST.(type) {
- case *redshiftast.InsertStmt, *redshiftast.UpdateStmt, *redshiftast.DeleteStmt, *redshiftast.MergeStmt:
- sampleDMLs = append(sampleDMLs, strings.TrimSpace(stmt.Text))
- default:
+ s, ok := c.Val.(*redshiftast.String)
+ if !ok || s.Str == "" || s.Str == "$user" {
+ continue
}
+ schemas = append(schemas, s.Str)
}
- return sampleDMLs, nil
+ return schemas, len(schemas) > 0
}
diff --git a/backend/plugin/parser/redshift/resource_change_test.go b/backend/plugin/parser/redshift/resource_change_test.go
index 037775d939..1336a4e752 100644
--- a/backend/plugin/parser/redshift/resource_change_test.go
+++ b/backend/plugin/parser/redshift/resource_change_test.go
@@ -1,6 +1,8 @@
package redshift
import (
+ "slices"
+ "strings"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
@@ -12,83 +14,149 @@ import (
)
func TestExtractChangedResources(t *testing.T) {
- statement := `CREATE TABLE t1 (c1 INT);
+ type table struct {
+ database string
+ schema string
+ name string
+ affected bool
+ }
+ for _, tc := range []struct {
+ name string
+ statement string
+ tables []table
+ dmlStatements []string
+ dmlCount int
+ insertCount int
+ }{
+ {
+ name: "ddl_affects_a_table_that_dml_also_changes",
+ statement: `CREATE TABLE t1 (c1 INT);
ALTER TABLE t1 ADD COLUMN c2 INT;
INSERT INTO t1 VALUES (1);
-UPDATE t1 SET c2 = 2;`
-
- dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)
- changedResources := model.NewChangedResources(dbMetadata)
- changedResources.AddTable(
- "db",
- "public",
- &storepb.ChangedResourceTable{
- Name: "t1",
+UPDATE t1 SET c2 = 2;`,
+ tables: []table{{database: "db", schema: "public", name: "t1", affected: true}},
+ dmlStatements: []string{"UPDATE t1 SET c2 = 2;"},
+ dmlCount: 1,
+ insertCount: 1,
},
- true,
- )
- want := &base.ChangeSummary{
- ChangedResources: changedResources,
- SampleDMLS: []string{
- "INSERT INTO t1 VALUES (1);",
- "UPDATE t1 SET c2 = 2;",
+ {
+ name: "dml_targets_are_not_affected",
+ statement: `UPDATE t SET c = 1 WHERE id = 1;
+DELETE FROM t WHERE id = 2;`,
+ tables: []table{{database: "db", schema: "public", name: "t"}},
+ dmlStatements: []string{
+ "UPDATE t SET c = 1 WHERE id = 1;",
+ "DELETE FROM t WHERE id = 2;",
+ },
+ dmlCount: 2,
},
- DMLCount: 2,
- InsertCount: 1,
- }
-
- stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, statement)
- require.NoError(t, err)
- got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), statement)
- require.NoError(t, err)
- require.Equal(t, want, got)
-}
-
-func TestExtractChangedResourcesSearchPathDropAndSelectInto(t *testing.T) {
- statement := `SET search_path TO analytics;
+ {
+ name: "insert_values_counts_rows_without_a_sample",
+ statement: `INSERT INTO t VALUES (1), (2), (3);
+INSERT INTO t (c) VALUES (4);
+INSERT INTO t DEFAULT VALUES;`,
+ tables: []table{{database: "db", schema: "public", name: "t"}},
+ insertCount: 5,
+ },
+ {
+ name: "insert_select_is_sampled",
+ statement: `INSERT INTO t SELECT * FROM s WHERE c > 1;`,
+ tables: []table{{database: "db", schema: "public", name: "t"}},
+ dmlStatements: []string{"INSERT INTO t SELECT * FROM s WHERE c > 1;"},
+ dmlCount: 1,
+ },
+ {
+ name: "merge_is_counted_without_a_sample",
+ statement: `MERGE INTO m USING s ON m.id = s.id WHEN MATCHED THEN UPDATE SET c = s.c;
+UPDATE t SET c = 1 WHERE id = 1;`,
+ tables: []table{{database: "db", schema: "public", name: "m"}, {database: "db", schema: "public", name: "t"}},
+ dmlStatements: []string{"UPDATE t SET c = 1 WHERE id = 1;"},
+ dmlCount: 2,
+ },
+ {
+ name: "truncate_affects_tables_resolved_against_the_search_path",
+ statement: `TRUNCATE a;
+SET search_path TO analytics;
+TRUNCATE TABLE b, public.c, other.s.d;
+RESET search_path;
+TRUNCATE e;`,
+ tables: []table{
+ {database: "db", schema: "public", name: "a", affected: true},
+ {database: "db", schema: "analytics", name: "b", affected: true},
+ {database: "db", schema: "public", name: "c", affected: true},
+ {database: "other", schema: "s", name: "d", affected: true},
+ {database: "db", schema: "public", name: "e", affected: true},
+ },
+ },
+ {
+ name: "search_path_drop_and_select_into",
+ statement: `SET search_path TO analytics;
CREATE TABLE unqualified(id INT);
SELECT * INTO copied_rows FROM public.rows;
DROP TABLE old_rows;
-DELETE FROM copied_rows WHERE id = 1;`
-
- dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)
- changedResources := model.NewChangedResources(dbMetadata)
- changedResources.AddTable(
- "db",
- "analytics",
- &storepb.ChangedResourceTable{
- Name: "unqualified",
+DELETE FROM copied_rows WHERE id = 1;`,
+ tables: []table{
+ {database: "db", schema: "analytics", name: "unqualified"},
+ {database: "db", schema: "analytics", name: "copied_rows"},
+ {database: "db", schema: "analytics", name: "old_rows", affected: true},
+ },
+ dmlStatements: []string{"SET LOCAL search_path TO \"analytics\";\nDELETE FROM copied_rows WHERE id = 1;"},
+ dmlCount: 1,
},
- false,
- )
- changedResources.AddTable(
- "db",
- "analytics",
- &storepb.ChangedResourceTable{
- Name: "copied_rows",
+ {
+ name: "sampled_dml_replays_the_search_path",
+ statement: `SET search_path TO '$user', analytics, public;
+UPDATE t SET c = 1;
+RESET search_path;
+UPDATE t SET c = 2;`,
+ tables: []table{
+ {database: "db", schema: "analytics", name: "t"},
+ {database: "db", schema: "public", name: "t"},
+ },
+ dmlStatements: []string{
+ "SET LOCAL search_path TO \"analytics\", \"public\";\nUPDATE t SET c = 1;",
+ "UPDATE t SET c = 2;",
+ },
+ dmlCount: 2,
},
- true,
- )
- changedResources.AddTable(
- "db",
- "analytics",
- &storepb.ChangedResourceTable{
- Name: "old_rows",
+ {
+ name: "comment_before_semicolon_between_ddl_and_dml",
+ statement: `ALTER TABLE a ADD COLUMN c INT /* note */;
+INSERT INTO b VALUES (1);
+DROP TABLE c;`,
+ tables: []table{
+ {database: "db", schema: "public", name: "a", affected: true},
+ {database: "db", schema: "public", name: "b"},
+ {database: "db", schema: "public", name: "c", affected: true},
+ },
+ insertCount: 1,
},
- true,
- )
- want := &base.ChangeSummary{
- ChangedResources: changedResources,
- SampleDMLS: []string{
- "DELETE FROM copied_rows WHERE id = 1;",
+ {
+ name: "every_dml_statement_is_listed",
+ statement: strings.Repeat("UPDATE t SET c = 1;\n", 12),
+ tables: []table{{database: "db", schema: "public", name: "t"}},
+ dmlStatements: slices.Repeat([]string{"UPDATE t SET c = 1;"}, 12),
+ dmlCount: 12,
},
- DMLCount: 1,
- InsertCount: 0,
- }
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_REDSHIFT, true /* caseSensitive */)
+ changedResources := model.NewChangedResources(dbMetadata)
+ for _, table := range tc.tables {
+ changedResources.AddTable(table.database, table.schema, &storepb.ChangedResourceTable{Name: table.name}, table.affected)
+ }
+ want := &base.ChangeSummary{
+ ChangedResources: changedResources,
+ DMLStatements: tc.dmlStatements,
+ DMLCount: tc.dmlCount,
+ InsertCount: tc.insertCount,
+ }
- stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, statement)
- require.NoError(t, err)
- got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), statement)
- require.NoError(t, err)
- require.Equal(t, want, got)
+ stmts, err := base.ParseStatements(storepb.Engine_REDSHIFT, tc.statement)
+ require.NoError(t, err)
+ got, err := base.ExtractChangedResources(storepb.Engine_REDSHIFT, "db", "public", dbMetadata, base.ExtractASTs(stmts), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+ })
+ }
}
diff --git a/backend/plugin/parser/redshift/statement_type.go b/backend/plugin/parser/redshift/statement_type.go
index 526f9b3b24..c4664fdff4 100644
--- a/backend/plugin/parser/redshift/statement_type.go
+++ b/backend/plugin/parser/redshift/statement_type.go
@@ -115,6 +115,8 @@ func classifyOmniStatementType(node redshiftast.Node) storepb.StatementType {
return storepb.StatementType_UPDATE
case *redshiftast.DeleteStmt:
return storepb.StatementType_DELETE
+ case *redshiftast.MergeStmt:
+ return storepb.StatementType_MERGE
case *redshiftast.RedshiftObjectStmt:
return classifyOmniRedshiftObjectStatementType(n.Command, n.ObjectType)
default:
diff --git a/backend/plugin/parser/redshift/statement_type_test.go b/backend/plugin/parser/redshift/statement_type_test.go
index 813aca0fe6..3d52ff5f9e 100644
--- a/backend/plugin/parser/redshift/statement_type_test.go
+++ b/backend/plugin/parser/redshift/statement_type_test.go
@@ -120,6 +120,13 @@ func TestGetStatementTypes(t *testing.T) {
storepb.StatementType_DELETE,
},
},
+ {
+ name: "merge",
+ statement: "MERGE INTO t1 USING t2 ON t1.id = t2.id WHEN MATCHED THEN DELETE;",
+ want: []storepb.StatementType{
+ storepb.StatementType_MERGE,
+ },
+ },
{
name: "truncate table",
statement: "TRUNCATE TABLE t1;",
diff --git a/backend/plugin/parser/tidb/resource_change.go b/backend/plugin/parser/tidb/resource_change.go
index d5f3d3ec38..a5c07a6e09 100644
--- a/backend/plugin/parser/tidb/resource_change.go
+++ b/backend/plugin/parser/tidb/resource_change.go
@@ -4,10 +4,10 @@ import (
"strings"
"unicode"
+ omniast "github.com/bytebase/omni/tidb/ast"
tidbast "github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -19,17 +19,24 @@ func init() {
}
func extractChangedResources(database string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {
- changedResources := model.NewChangedResources(dbMetadata)
- dmlCount := 0
- insertCount := 0
- var sampleDMLs []string
+ summary := &base.ChangeSummary{
+ ChangedResources: model.NewChangedResources(dbMetadata),
+ }
for _, ast := range asts {
tidbAST, ok := GetTiDBAST(ast)
if !ok {
- return nil, errors.New("expected TiDB AST")
+ // The pingcap re-parse rejects some statements omni accepts. Read those from the
+ // omni AST rather than failing the summary of every other statement.
+ omniAST, ok := ast.(*OmniAST)
+ if !ok {
+ return nil, errors.New("expected TiDB AST")
+ }
+ addOmniTables(database, omniAST.Node, summary.ChangedResources)
+ countOmniDML(summary, omniAST)
+ continue
}
node := tidbAST.Node
- err := getResourceChanges(database, node, statement, changedResources)
+ err := getResourceChanges(database, node, statement, summary.ChangedResources)
if err != nil {
return nil, err
}
@@ -37,29 +44,128 @@ func extractChangedResources(database string, _ string, dbMetadata *model.Databa
switch n := node.(type) {
case *tidbast.InsertStmt:
if len(n.Lists) > 0 {
- insertCount += len(n.Lists)
+ summary.InsertCount += len(n.Lists)
continue
}
-
- dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, trimStatement(n.Text()))
- }
+ addSampleDML(summary, n.Text())
case *tidbast.UpdateStmt, *tidbast.DeleteStmt:
- dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, trimStatement(node.Text()))
+ addSampleDML(summary, node.Text())
+ case *tidbast.NonTransactionalDMLStmt:
+ if n.DryRun == tidbast.NoDryRun {
+ addBatchSampleDML(summary, ast)
}
default:
}
}
+ return summary, nil
+}
- return &base.ChangeSummary{
- ChangedResources: changedResources,
- DMLCount: dmlCount,
- SampleDMLS: sampleDMLs,
- InsertCount: insertCount,
- }, nil
+// addOmniTables records the tables an omni INSERT, UPDATE, DELETE, or BATCH statement names, as
+// getResourceChanges does for the pingcap AST.
+func addOmniTables(database string, node omniast.Node, changedResources *model.ChangedResources) {
+ var tableExprs []omniast.TableExpr
+ switch n := node.(type) {
+ case *omniast.InsertStmt:
+ if n.Table != nil {
+ tableExprs = append(tableExprs, n.Table)
+ }
+ case *omniast.UpdateStmt:
+ tableExprs = n.Tables
+ case *omniast.DeleteStmt:
+ tableExprs = append(append(tableExprs, n.Tables...), n.Using...)
+ case *omniast.BatchStmt:
+ if n.DryRun == omniast.BatchDryRunNone {
+ addOmniTables(database, n.DML, changedResources)
+ }
+ return
+ default:
+ return
+ }
+ var add func(expr omniast.TableExpr)
+ add = func(expr omniast.TableExpr) {
+ switch e := expr.(type) {
+ case *omniast.TableRef:
+ d := e.Schema
+ if d == "" {
+ d = database
+ }
+ changedResources.AddTable(d, "", &storepb.ChangedResourceTable{Name: e.Name}, false)
+ case *omniast.JoinClause:
+ add(e.Left)
+ add(e.Right)
+ default:
+ }
+ }
+ for _, expr := range tableExprs {
+ add(expr)
+ }
+}
+
+func countOmniDML(summary *base.ChangeSummary, omniAST *OmniAST) {
+ switch n := omniAST.Node.(type) {
+ case *omniast.InsertStmt:
+ switch {
+ case len(n.Values) > 0:
+ summary.InsertCount += len(n.Values)
+ case len(n.SetList) > 0:
+ summary.InsertCount++
+ default:
+ addSampleDML(summary, omniAST.Text)
+ }
+ case *omniast.UpdateStmt, *omniast.DeleteStmt:
+ addSampleDML(summary, omniAST.Text)
+ case *omniast.BatchStmt:
+ if n.DryRun == omniast.BatchDryRunNone {
+ addBatchSampleDML(summary, omniAST)
+ }
+ default:
+ }
+}
+
+// addBatchSampleDML samples the DML inside a BATCH statement, because EXPLAIN rejects the
+// BATCH syntax. A BATCH without an omni AST is counted but not sampled.
+func addBatchSampleDML(summary *base.ChangeSummary, ast base.AST) {
+ omniAST, ok := ast.(*OmniAST)
+ if !ok {
+ summary.DMLCount++
+ return
+ }
+ batch, ok := omniAST.Node.(*omniast.BatchStmt)
+ if !ok {
+ summary.DMLCount++
+ return
+ }
+ // omni positions the text after an executable comment as if the comment's markers were removed.
+ if strings.Contains(omniAST.Text, "/*!") || strings.Contains(omniAST.Text, "/*T!") || strings.Contains(omniAST.Text, "/*M!") {
+ summary.DMLCount++
+ return
+ }
+ var loc omniast.Loc
+ switch dml := batch.DML.(type) {
+ case *omniast.DeleteStmt:
+ loc = dml.Loc
+ case *omniast.UpdateStmt:
+ loc = dml.Loc
+ case *omniast.InsertStmt:
+ loc = dml.Loc
+ default:
+ summary.DMLCount++
+ return
+ }
+ end := loc.End
+ if end <= loc.Start || end > len(omniAST.Text) {
+ end = len(omniAST.Text)
+ }
+ if loc.Start <= 0 || loc.Start >= end {
+ summary.DMLCount++
+ return
+ }
+ addSampleDML(summary, omniAST.Text[loc.Start:end])
+}
+
+func addSampleDML(summary *base.ChangeSummary, text string) {
+ summary.DMLCount++
+ summary.DMLStatements = append(summary.DMLStatements, trimStatement(text))
}
// addTiDBObjectDatabase records a database-only write target for a non-table object DDL
@@ -293,6 +399,10 @@ func getResourceChanges(database string, node tidbast.StmtNode, _ string, change
)
}
}
+ case *tidbast.NonTransactionalDMLStmt:
+ if node.DryRun == tidbast.NoDryRun {
+ return getResourceChanges(database, node.DMLStmt, "", changedResources)
+ }
default:
}
return nil
diff --git a/backend/plugin/parser/tidb/resource_change_test.go b/backend/plugin/parser/tidb/resource_change_test.go
index 056eb88ae8..041236ef16 100644
--- a/backend/plugin/parser/tidb/resource_change_test.go
+++ b/backend/plugin/parser/tidb/resource_change_test.go
@@ -37,7 +37,7 @@ func TestExtractChangedResources(t *testing.T) {
)
want := &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: []string{
+ DMLStatements: []string{
"UPDATE t1 SET c1 = 5",
},
DMLCount: 1,
@@ -106,3 +106,66 @@ func TestExtractChangedResourcesObjectDDLDatabaseOnly(t *testing.T) {
require.NoError(t, err)
require.Empty(t, got.ChangedResources.GetDatabaseOnlyTargets())
}
+
+func TestExtractChangedResourcesDMLCounts(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ dmlCount int
+ insertCount int
+ samples []string
+ tables []string
+ }{
+ {
+ name: "insert_set_is_one_row",
+ statement: `INSERT INTO t SET c = 1, d = 2;`,
+ insertCount: 1,
+ tables: []string{"t"},
+ },
+ {
+ name: "batch_samples_inner_dml",
+ statement: `BATCH ON id LIMIT 1000 DELETE FROM t WHERE c = 1;`,
+ dmlCount: 1,
+ samples: []string{"DELETE FROM t WHERE c = 1"},
+ tables: []string{"t"},
+ },
+ {
+ name: "batch_with_an_executable_comment_is_counted_without_a_sample",
+ statement: `BATCH ON id LIMIT 1000 DELETE FROM t WHERE /*T![clustered_index] c = 0 AND */ id < 10000000000;`,
+ dmlCount: 1,
+ tables: []string{"t"},
+ },
+ {
+ name: "batch_dry_run_changes_nothing",
+ statement: `BATCH ON id LIMIT 1000 DRY RUN DELETE FROM t WHERE c = 1;`,
+ },
+ {
+ // pingcap cannot parse the row alias, which omni accepts.
+ name: "statement_without_pingcap_ast",
+ statement: "INSERT INTO t VALUES (1, 2) AS new ON DUPLICATE KEY UPDATE c = new.c;\nUPDATE t2 SET c = 1 WHERE id = 1;",
+ dmlCount: 1,
+ insertCount: 1,
+ samples: []string{"UPDATE t2 SET c = 1 WHERE id = 1"},
+ tables: []string{"t", "t2"},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ stmts, err := base.ParseStatements(storepb.Engine_TIDB, tc.statement)
+ require.NoError(t, err)
+ got, err := extractChangedResources("db", "", nil /* dbMetadata */, base.ExtractASTs(stmts), tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, tc.dmlCount, got.DMLCount)
+ require.Equal(t, tc.insertCount, got.InsertCount)
+ require.Equal(t, tc.samples, got.DMLStatements)
+ var tables []string
+ for _, database := range got.ChangedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ tables = append(tables, table.GetName())
+ }
+ }
+ }
+ require.Equal(t, tc.tables, tables)
+ })
+ }
+}
diff --git a/backend/plugin/parser/tsql/resource_change.go b/backend/plugin/parser/tsql/resource_change.go
index 08ebd6af4c..b39ed043f0 100644
--- a/backend/plugin/parser/tsql/resource_change.go
+++ b/backend/plugin/parser/tsql/resource_change.go
@@ -7,7 +7,6 @@ import (
"github.com/bytebase/omni/mssql/ast"
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
@@ -22,13 +21,11 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMet
changedResources := model.NewChangedResources(dbMetadata)
var dmlCount, insertCount int
- var sampleDMLs []string
+ var dmlStatements []string
addDML := func(text string) {
dmlCount++
- if len(sampleDMLs) < common.MaximumLintExplainSize {
- sampleDMLs = append(sampleDMLs, trimStatement(text))
- }
+ dmlStatements = append(dmlStatements, trimStatement(text))
}
addTable := func(ref *ast.TableRef, affectData bool) {
@@ -116,6 +113,12 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMet
insertCount += vc.Rows.Len()
continue
}
+ // SHOWPLAN estimates one row for INSERT ... EXEC whatever the procedure returns, so the
+ // statement is counted without an estimate.
+ if _, ok := n.Source.(*ast.ExecStmt); ok {
+ dmlCount++
+ continue
+ }
addDML(omniAST.Text)
case *ast.UpdateStmt:
@@ -170,7 +173,7 @@ func extractChangedResources(currentDatabase string, currentSchema string, dbMet
return &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: sampleDMLs,
+ DMLStatements: dmlStatements,
DMLCount: dmlCount,
InsertCount: insertCount,
}, nil
diff --git a/backend/plugin/parser/tsql/resource_change_test.go b/backend/plugin/parser/tsql/resource_change_test.go
index 9a7ddab9e8..502c78bba8 100644
--- a/backend/plugin/parser/tsql/resource_change_test.go
+++ b/backend/plugin/parser/tsql/resource_change_test.go
@@ -28,7 +28,7 @@ func TestExtractChangedResources(t *testing.T) {
)
want := &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: []string{
+ DMLStatements: []string{
"UPDATE t1 SET c1 = 5;",
},
DMLCount: 1,
@@ -68,7 +68,20 @@ func TestExtractChangedResources_InsertDefaultValues(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 1, got.InsertCount)
require.Equal(t, 0, got.DMLCount)
- require.Empty(t, got.SampleDMLS)
+ require.Empty(t, got.DMLStatements)
+}
+
+func TestExtractChangedResources_InsertExec(t *testing.T) {
+ const statement = `INSERT INTO dbo.t1 EXEC dbo.sp;`
+
+ stmts, err := base.ParseStatements(storepb.Engine_MSSQL, statement)
+ require.NoError(t, err)
+ asts := base.ExtractASTs(stmts)
+ got, err := extractChangedResources("DB", "dbo", nil, asts, statement)
+ require.NoError(t, err)
+ require.Equal(t, 1, got.DMLCount)
+ require.Empty(t, got.DMLStatements)
+ require.Zero(t, got.InsertCount)
}
func TestExtractChangedResources_Truncate(t *testing.T) {
diff --git a/backend/runner/plancheck/statement_report_executor.go b/backend/runner/plancheck/statement_report_executor.go
index 2d66c6f731..041b8ad260 100644
--- a/backend/runner/plancheck/statement_report_executor.go
+++ b/backend/runner/plancheck/statement_report_executor.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
+ "strings"
"github.com/pkg/errors"
@@ -11,7 +12,9 @@ import (
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/dbfactory"
"github.com/bytebase/bytebase/backend/component/sheet"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
"github.com/bytebase/bytebase/backend/plugin/db"
+ cockroachdbdriver "github.com/bytebase/bytebase/backend/plugin/db/cockroachdb"
mssqldriver "github.com/bytebase/bytebase/backend/plugin/db/mssql"
mysqldriver "github.com/bytebase/bytebase/backend/plugin/db/mysql"
oracledriver "github.com/bytebase/bytebase/backend/plugin/db/oracle"
@@ -100,7 +103,7 @@ func (e *StatementReportExecutor) RunForTarget(ctx context.Context, target *Chec
Code: common.Ok.Int32(),
Title: "OK",
}
- summaryReport, err := GetSQLSummaryReport(ctx, e.store, e.sheetManager, e.dbFactory, database, fullSheet.Statement)
+ summaryReport, estimateWarning, err := GetSQLSummaryReport(ctx, e.store, e.sheetManager, e.dbFactory, database, fullSheet.Statement)
if err != nil {
return nil, err
}
@@ -109,17 +112,28 @@ func (e *StatementReportExecutor) RunForTarget(ctx context.Context, target *Chec
SqlSummaryReport: summaryReport,
}
}
+ if estimateWarning != "" {
+ planCheckRunResult.Status = storepb.Advice_WARNING
+ planCheckRunResult.Code = code.StatementExplainQueryFailed.Int32()
+ planCheckRunResult.Title = AffectedRowsEstimateIncompleteTitle
+ planCheckRunResult.Content = estimateWarning
+ }
return []*storepb.PlanCheckRunResult_Result{planCheckRunResult}, nil
}
-// GetSQLSummaryReport gets the SQL summary report for the given statement and database.
-func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager *sheet.Manager, dbFactory *dbfactory.DBFactory, database *store.DatabaseMessage, statement string) (*storepb.PlanCheckRunResult_Result_SqlSummaryReport, error) {
+// AffectedRowsEstimateIncompleteTitle titles the warning for DML statements whose affected
+// rows could not be estimated.
+const AffectedRowsEstimateIncompleteTitle = "Affected rows estimate is incomplete"
+
+// GetSQLSummaryReport gets the SQL summary report for the given statement and database. The
+// returned warning is non-empty when some DML statements' affected rows could not be estimated.
+func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager *sheet.Manager, dbFactory *dbfactory.DBFactory, database *store.DatabaseMessage, statement string) (*storepb.PlanCheckRunResult_Result_SqlSummaryReport, string, error) {
instance, err := stores.GetInstanceByResourceID(ctx, database.InstanceID)
if err != nil {
- return nil, err
+ return nil, "", err
}
if instance == nil {
- return nil, errors.Errorf("instance not found: %s", database.InstanceID)
+ return nil, "", errors.Errorf("instance not found: %s", database.InstanceID)
}
databaseSchema, err := stores.GetDBSchema(ctx, &store.FindDBSchemaMessage{
Workspace: instance.Workspace,
@@ -127,19 +141,19 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager
DatabaseName: database.DatabaseName,
})
if err != nil {
- return nil, err
+ return nil, "", err
}
if databaseSchema == nil {
- return nil, errors.Errorf("database schema %s not found", database.String())
+ return nil, "", errors.Errorf("database schema %s not found", database.String())
}
if databaseSchema.GetProto() == nil {
- return nil, errors.Errorf("database schema metadata %s not found", database.String())
+ return nil, "", errors.Errorf("database schema metadata %s not found", database.String())
}
stmts, syntaxAdvices := sheetManager.GetStatementsForChecks(instance.Metadata.GetEngine(), statement)
if len(syntaxAdvices) > 0 {
// Return nil as it should already be checked before running this function.
- return nil, nil
+ return nil, "", nil
}
asts := parserbase.ExtractASTs(stmts)
@@ -147,13 +161,13 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager
var defaultSchema string
project, err := stores.GetProject(ctx, &store.FindProjectMessage{Workspace: instance.Workspace, ResourceID: &database.ProjectID})
if err != nil {
- return nil, err
+ return nil, "", err
}
driver, err := dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{
TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),
})
if err != nil {
- return nil, err
+ return nil, "", err
}
defer driver.Close(ctx)
@@ -161,7 +175,7 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager
case storepb.Engine_POSTGRES:
pd, ok := driver.(*pgdriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid pg driver type")
+ return nil, "", errors.Errorf("invalid pg driver type")
}
explainCalculator = pd.CountAffectedRows
// Empty so pg's extractChangedResources falls back to the database's actual
@@ -170,46 +184,54 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager
case storepb.Engine_REDSHIFT:
rd, ok := driver.(*redshiftdriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid redshift driver type")
+ return nil, "", errors.Errorf("invalid redshift driver type")
}
explainCalculator = rd.CountAffectedRows
defaultSchema = "public"
+ case storepb.Engine_COCKROACHDB:
+ cd, ok := driver.(*cockroachdbdriver.Driver)
+ if !ok {
+ return nil, "", errors.Errorf("invalid cockroachdb driver type")
+ }
+ explainCalculator = cd.CountAffectedRows
+ // Empty so the extractor resolves names with the database's synced search_path, as for PostgreSQL.
+ defaultSchema = ""
case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:
md, ok := driver.(*mysqldriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid mysql driver type")
+ return nil, "", errors.Errorf("invalid mysql driver type")
}
explainCalculator = md.CountAffectedRows
defaultSchema = ""
case storepb.Engine_TIDB:
md, ok := driver.(*tidbdriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid tidb driver type")
+ return nil, "", errors.Errorf("invalid tidb driver type")
}
explainCalculator = md.CountAffectedRows
defaultSchema = ""
case storepb.Engine_ORACLE:
od, ok := driver.(*oracledriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid oracle driver type")
+ return nil, "", errors.Errorf("invalid oracle driver type")
}
explainCalculator = od.CountAffectedRows
defaultSchema = database.DatabaseName
case storepb.Engine_MSSQL:
md, ok := driver.(*mssqldriver.Driver)
if !ok {
- return nil, errors.Errorf("invalid mssql driver type")
+ return nil, "", errors.Errorf("invalid mssql driver type")
}
explainCalculator = md.CountAffectedRows
defaultSchema = "dbo"
default:
// Already checked in the Run().
- return nil, nil
+ return nil, "", nil
}
sqlTypes, err := SummaryStatementTypes(instance.Metadata.GetEngine(), asts)
if err != nil {
- return nil, err
+ return nil, "", err
}
// Database secrets feature has been removed
@@ -217,15 +239,15 @@ func GetSQLSummaryReport(ctx context.Context, stores *store.Store, sheetManager
// Database secrets feature removed - using original statement directly
changeSummary, err := parserbase.ExtractChangedResources(instance.Metadata.GetEngine(), database.DatabaseName, defaultSchema, databaseSchema, asts, statement)
if err != nil {
- return nil, errors.Wrapf(err, "failed to extract changed resources")
+ return nil, "", errors.Wrapf(err, "failed to extract changed resources")
}
- totalAffectedRows := calculateAffectedRows(ctx, changeSummary, explainCalculator)
+ totalAffectedRows, estimateWarning := calculateAffectedRows(ctx, instance.Metadata.GetEngine(), changeSummary, explainCalculator)
return &storepb.PlanCheckRunResult_Result_SqlSummaryReport{
StatementTypes: sqlTypes,
AffectedRows: totalAffectedRows,
ChangedResources: changeSummary.ChangedResources.Build(),
- }, nil
+ }, estimateWarning, nil
}
// SummaryStatementTypes classifies a summary report's statements through the
@@ -244,27 +266,326 @@ func SummaryStatementTypes(engine storepb.Engine, asts []parserbase.AST) ([]stor
type getAffectedRowsFromExplain func(context.Context, string) (int64, error)
-func calculateAffectedRows(ctx context.Context, changeSummary *parserbase.ChangeSummary, explainCalculator getAffectedRowsFromExplain) int64 {
- var totalAffectedRows int64
- // Count DMLs.
- sampleCount := 0
+// calculateAffectedRows adds the estimated rows of the DML statements, the inserted VALUES rows,
+// and the rows of tables changed by DDL. The returned warning describes DML statements whose rows
+// could not be estimated.
+func calculateAffectedRows(ctx context.Context, engine storepb.Engine, changeSummary *parserbase.ChangeSummary, explainCalculator getAffectedRowsFromExplain) (int64, string) {
+ mysqlFamily := engine == storepb.Engine_MYSQL || engine == storepb.Engine_MARIADB || engine == storepb.Engine_TIDB || engine == storepb.Engine_OCEANBASE
+ shapes := groupStatementsByShape(changeSummary.DMLStatements, mysqlFamily)
+ sampled := 0
+ var failures []error
if explainCalculator != nil {
- for _, dml := range changeSummary.SampleDMLS {
- count, err := explainCalculator(ctx, dml)
- if err != nil {
- slog.Error("failed to calculate affected rows", log.BBError(err))
- continue
+ // Each round estimates the next statement of every shape, so the samples cover as many
+ // shapes as they can before sampling a shape twice.
+ for round := 0; sampled < common.MaximumLintExplainSize; round++ {
+ sampledInRound := false
+ for _, shape := range shapes {
+ if round >= len(shape.statements) || sampled >= common.MaximumLintExplainSize {
+ continue
+ }
+ sampledInRound = true
+ sampled++
+ count, err := explainCalculator(ctx, shape.statements[round])
+ if err != nil {
+ slog.Error("failed to calculate affected rows", log.BBError(err))
+ failures = append(failures, err)
+ shape.failed++
+ continue
+ }
+ shape.rows = common.AddRows(shape.rows, count)
+ shape.estimated++
}
- sampleCount++
- totalAffectedRows += count
+ if !sampledInRound {
+ break
+ }
+ }
+ }
+
+ var dmlRows, estimatedRows int64
+ estimated := 0
+ // The unestimated statements of a sampled shape count as that shape's average estimate; the
+ // statements of other shapes, and DML statements without text, count as the overall average and
+ // are reported as not estimated.
+ unestimated := max(changeSummary.DMLCount-len(changeSummary.DMLStatements), 0)
+ for _, shape := range shapes {
+ estimatedRows = common.AddRows(estimatedRows, shape.rows)
+ estimated += shape.estimated
+ switch {
+ case shape.estimated == len(shape.statements):
+ dmlRows = common.AddRows(dmlRows, shape.rows)
+ case shape.estimated > 0:
+ dmlRows = common.AddRows(dmlRows, common.RoundRows(float64(shape.rows)/float64(shape.estimated)*float64(len(shape.statements))))
+ default:
+ unestimated += len(shape.statements)
+ }
+ }
+ if estimated > 0 && unestimated > 0 {
+ dmlRows = common.AddRows(dmlRows, common.RoundRows(float64(estimatedRows)/float64(estimated)*float64(unestimated)))
+ }
+ totalAffectedRows := common.AddRows(dmlRows, int64(changeSummary.InsertCount))
+ totalAffectedRows = common.AddRows(totalAffectedRows, changeSummary.ChangedResources.CountAffectedTableRows())
+
+ // A failed sample of a shape with other estimates counts as that shape's average, but is still
+ // not estimated.
+ notEstimated := unestimated
+ for _, shape := range shapes {
+ if shape.estimated > 0 {
+ notEstimated += shape.failed
+ }
+ }
+ var warning string
+ switch {
+ case len(failures) > 0:
+ warning = fmt.Sprintf("Affected rows could not be estimated for %d of %d DML statements: %v", notEstimated, changeSummary.DMLCount, failures[0])
+ case notEstimated > 0:
+ warning = fmt.Sprintf("Affected rows could not be estimated for %d of %d DML statements.", notEstimated, changeSummary.DMLCount)
+ default:
+ }
+ return totalAffectedRows, warning
+}
+
+// statementShape is the DML statements that share a shape, with the estimates of those sampled.
+type statementShape struct {
+ statements []string
+ estimated int
+ failed int
+ rows int64
+}
+
+// groupStatementsByShape groups statements that differ only in literal values or spacing, in the
+// order each shape first appears.
+func groupStatementsByShape(statements []string, mysqlFamily bool) []*statementShape {
+ var shapes []*statementShape
+ byKey := map[string]*statementShape{}
+ for _, statement := range statements {
+ key := shapeKey(statement, mysqlFamily)
+ shape, ok := byKey[key]
+ if !ok {
+ shape = &statementShape{}
+ byKey[key] = shape
+ shapes = append(shapes, shape)
}
+ shape.statements = append(shape.statements, statement)
}
- if sampleCount > 0 {
- totalAffectedRows = int64((float64(totalAffectedRows) / float64(sampleCount)) * float64(changeSummary.DMLCount))
+ return shapes
+}
+
+// shapeKey replaces the string and numeric literals of a statement with ? and drops comments and
+// whitespace that does not separate words. Everything else keeps its text, including identifiers,
+// their case, double-quoted text, which MySQL's ANSI_QUOTES makes an identifier, and the hints and
+// executable comments that isKeptComment lists. A statement with text the scan does not follow, such
+// as an unclosed quote, an Oracle q'{...}' string, a dollar-quoted string, a nested comment, or a
+// quote in brackets, is its own key, because a quote in that text could make the scan read the
+// statement's clauses as a literal. mysqlFamily applies the MySQL, MariaDB, TiDB, and OceanBase rules
+// that a backslash escapes the next character of a quoted string, that # starts a line comment, and
+// that -- starts one only before whitespace or a control character.
+func shapeKey(statement string, mysqlFamily bool) string {
+ var b strings.Builder
+ space := false
+ // Whitespace is kept between words, where ? counts as a word, and in - - so that it never reads
+ // as a comment marker.
+ isWord := func(c byte) bool { return c == '?' || isWordByte(c) }
+ write := func(c byte) {
+ if space && b.Len() > 0 {
+ if last := b.String()[b.Len()-1]; (isWord(last) && isWord(c)) || (last == '-' && c == '-') {
+ b.WriteByte(' ')
+ }
+ }
+ space = false
+ b.WriteByte(c)
}
- totalAffectedRows += int64(changeSummary.InsertCount)
- // Count affected rows by DDLs.
- totalAffectedRows += changeSummary.ChangedResources.CountAffectedTableRows()
+ for i := 0; i < len(statement); {
+ c := statement[i]
+ switch {
+ case c == '\'':
+ word := wordBefore(statement, i)
+ if strings.EqualFold(word, "q") || strings.EqualFold(word, "nq") {
+ return statement
+ }
+ // PostgreSQL E'...' and CockroachDB b'...' strings escape with backslashes too.
+ end, closed := quotedEnd(statement, i, mysqlFamily || strings.EqualFold(word, "e") || strings.EqualFold(word, "b"))
+ if !closed {
+ return statement
+ }
+ i = end
+ write('?')
+ case c == '$' && !mysqlFamily && (i == 0 || !isWordByte(statement[i-1])) && isDollarQuoteStart(statement[i:]):
+ return statement
+ case c == '[':
+ // A SQL Server bracketed identifier, where ]] continues it, or an array subscript, keeps its text.
+ end := i + 1
+ for end < len(statement) && (statement[end] != ']' || (end+1 < len(statement) && statement[end+1] == ']')) {
+ if statement[end] == ']' {
+ end++
+ }
+ end++
+ }
+ if end >= len(statement) || strings.ContainsAny(statement[i+1:end], `'"`) {
+ return statement
+ }
+ write(c)
+ b.WriteString(statement[i+1 : end+1])
+ i = end + 1
+ case c == '"' || c == '`':
+ end, closed := quotedEnd(statement, i, mysqlFamily && c == '"')
+ if !closed {
+ return statement
+ }
+ write(c)
+ b.WriteString(statement[i+1 : end])
+ i = end
+ case (strings.HasPrefix(statement[i:], "--") && (!mysqlFamily || i+2 == len(statement) || statement[i+2] <= ' ')) ||
+ strings.HasPrefix(statement[i:], "/*") || (mysqlFamily && c == '#'):
+ end := commentEnd(statement, i)
+ if c == '/' && strings.Contains(statement[i+2:end], "/*") {
+ return statement
+ }
+ if isKeptComment(statement[i:end]) {
+ write(c)
+ b.WriteString(statement[i+1 : end])
+ } else {
+ space = true
+ }
+ i = end
+ case c >= '0' && c <= '9' && (i == 0 || !isWordByte(statement[i-1])):
+ if end := numberEnd(statement, i, mysqlFamily); end >= 0 {
+ i = end
+ write('?')
+ break
+ }
+ // A word that starts with digits but is not a number, such as [2024_orders], is an identifier.
+ for i < len(statement) && isWordByte(statement[i]) {
+ write(statement[i])
+ i++
+ }
+ case c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v':
+ space = true
+ i++
+ default:
+ write(c)
+ i++
+ }
+ }
+ return b.String()
+}
+
+// wordBefore returns the word that ends at offset end of statement.
+func wordBefore(statement string, end int) string {
+ start := end
+ for start > 0 && isWordByte(statement[start-1]) {
+ start--
+ }
+ return statement[start:end]
+}
+
+// isDollarQuoteStart reports whether text starts with a dollar-quote delimiter, such as $$ or $tag$.
+func isDollarQuoteStart(text string) bool {
+ end := 1
+ for end < len(text) && text[end] != '$' && isWordByte(text[end]) && (end > 1 || text[end] < '0' || text[end] > '9') {
+ end++
+ }
+ return end < len(text) && text[end] == '$'
+}
+
+// isKeptComment reports whether a comment changes how a statement runs: an optimizer hint, such as
+// /*+ ... */ or Oracle's --+ ..., or a MySQL, MariaDB, or TiDB executable comment, such as
+// /*!80000 ... */, /*M!100100 ... */, or /*T![feature] ... */.
+func isKeptComment(comment string) bool {
+ for _, prefix := range []string{"/*+", "--+", "/*!", "/*M!", "/*T!"} {
+ if strings.HasPrefix(comment, prefix) {
+ return true
+ }
+ }
+ return false
+}
+
+// commentEnd returns the offset just past the comment that starts at start: past the newline that
+// ends a -- or # comment, which keeps the next line out of the comment's copied text, or past the */
+// that closes a /* comment, or the end of the statement when it does not close.
+func commentEnd(statement string, start int) int {
+ if statement[start] != '/' {
+ if end := strings.IndexByte(statement[start:], '\n'); end >= 0 {
+ return start + end + 1
+ }
+ return len(statement)
+ }
+ if end := strings.Index(statement[start+2:], "*/"); end >= 0 {
+ return start + 2 + end + 2
+ }
+ return len(statement)
+}
+
+// numberEnd returns the offset just past the number that starts at start, or -1 when the word there
+// is not a number: a decimal, or a 0x hexadecimal or 0b binary integer. Outside mysqlFamily, where
+// words such as 0X1F, 0o17, and 1_000 are identifiers, it also reads uppercase prefixes, PostgreSQL's
+// 0o octal integers, and _ between digits.
+func numberEnd(statement string, start int, mysqlFamily bool) int {
+ isDigit := func(i int, digits string) bool {
+ return i < len(statement) && (strings.IndexByte(digits, statement[i]) >= 0 || (!mysqlFamily && statement[i] == '_'))
+ }
+ prefixDigits := ""
+ if start+1 < len(statement) && statement[start] == '0' {
+ switch c := statement[start+1]; {
+ case c == 'x' || (c == 'X' && !mysqlFamily):
+ prefixDigits = "0123456789abcdefABCDEF"
+ case c == 'b' || (c == 'B' && !mysqlFamily):
+ prefixDigits = "01"
+ case (c == 'o' || c == 'O') && !mysqlFamily:
+ prefixDigits = "01234567"
+ default:
+ }
+ }
+ i := start
+ if prefixDigits != "" {
+ i += 2
+ for isDigit(i, prefixDigits) {
+ i++
+ }
+ if i == start+2 {
+ return -1
+ }
+ } else {
+ for isDigit(i, "0123456789.") {
+ i++
+ }
+ if i < len(statement) && (statement[i] == 'e' || statement[i] == 'E') {
+ exponent := i + 1
+ if exponent < len(statement) && (statement[exponent] == '+' || statement[exponent] == '-') {
+ exponent++
+ }
+ if isDigit(exponent, "0123456789") {
+ i = exponent
+ for isDigit(i, "0123456789") {
+ i++
+ }
+ }
+ }
+ }
+ if i < len(statement) && isWordByte(statement[i]) {
+ return -1
+ }
+ return i
+}
+
+// quotedEnd returns the offset just past the quoted text that starts at start, where a doubled
+// quote continues the text, and so does an escaped quote with backslashEscapes. It reports false
+// when the text never closes.
+func quotedEnd(statement string, start int, backslashEscapes bool) (int, bool) {
+ quote := statement[start]
+ for i := start + 1; i < len(statement); i++ {
+ switch {
+ case backslashEscapes && statement[i] == '\\':
+ i++
+ case statement[i] != quote:
+ case i+1 < len(statement) && statement[i+1] == quote:
+ i++
+ default:
+ return i + 1, true
+ }
+ }
+ return len(statement), false
+}
- return totalAffectedRows
+func isWordByte(c byte) bool {
+ return c == '_' || c == '$' || c >= 0x80 || ('0' <= c && c <= '9') || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
}
diff --git a/backend/runner/plancheck/statement_report_executor_test.go b/backend/runner/plancheck/statement_report_executor_test.go
index fac96fe4c5..c80ee2e877 100644
--- a/backend/runner/plancheck/statement_report_executor_test.go
+++ b/backend/runner/plancheck/statement_report_executor_test.go
@@ -1,16 +1,308 @@
package plancheck
import (
+ "context"
+ "fmt"
+ "math"
"slices"
"testing"
+ "github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
)
+func TestCalculateAffectedRows(t *testing.T) {
+ type estimate struct {
+ statement string
+ rows int64
+ err error
+ }
+ repeat := func(count int, format string, rows int64) []estimate {
+ var estimates []estimate
+ for i := range count {
+ estimates = append(estimates, estimate{fmt.Sprintf(format, i+1), rows, nil})
+ }
+ return estimates
+ }
+ for _, tc := range []struct {
+ name string
+ statements []estimate
+ dmlCount int
+ insertCount int
+ wantRows int64
+ wantWarning string
+ wantExplains int
+ }{
+ {
+ name: "every_dml_estimated_sums_exactly",
+ statements: []estimate{{"u1", 10, nil}, {"u2", 17, nil}, {"u3", 34, nil}},
+ dmlCount: 3,
+ wantRows: 61,
+ wantExplains: 3,
+ },
+ {
+ name: "dml_without_text_counts_as_the_average_estimate",
+ statements: []estimate{{"u1", 10, nil}, {"u2", 30, nil}},
+ dmlCount: 5,
+ wantRows: 100,
+ wantWarning: "Affected rows could not be estimated for 3 of 5 DML statements.",
+ wantExplains: 2,
+ },
+ {
+ // The ten one-row updates share a shape, so the delete after them is still estimated.
+ name: "samples_cover_every_shape",
+ statements: append(repeat(10, "UPDATE t SET v = 'y' WHERE id = %d;", 1),
+ estimate{"DELETE FROM big WHERE s_id <= 50;", 5000, nil}),
+ dmlCount: 11,
+ wantRows: 5010,
+ wantExplains: 10,
+ },
+ {
+ // Five statements of each shape are estimated: 2 * 15 + 100 * 5.
+ name: "shape_average_counts_for_its_unsampled_statements",
+ statements: append(repeat(15, "UPDATE t SET c = c + 1 WHERE id = %d;", 2),
+ repeat(5, "DELETE FROM big WHERE s_id = %d;", 100)...),
+ dmlCount: 20,
+ wantRows: 530,
+ wantExplains: 10,
+ },
+ {
+ // The first ten shapes are estimated at 10 rows each; the other two count as that average.
+ name: "shapes_beyond_the_samples_count_as_the_average_estimate",
+ statements: append(repeat(10, "UPDATE t%d SET c = 1;", 10), estimate{"DELETE FROM a;", 1000, nil}, estimate{"DELETE FROM b;", 1000, nil}),
+ dmlCount: 12,
+ wantRows: 120,
+ wantWarning: "Affected rows could not be estimated for 2 of 12 DML statements.",
+ wantExplains: 10,
+ },
+ {
+ name: "failed_sample_warns",
+ statements: []estimate{{"u1", 3, nil}, {"d1", 0, errors.New("table t_new does not exist")}},
+ dmlCount: 2,
+ insertCount: 4,
+ wantRows: 10,
+ wantWarning: "Affected rows could not be estimated for 1 of 2 DML statements: table t_new does not exist",
+ wantExplains: 2,
+ },
+ {
+ // The shape's other statement is estimated, so the failed one counts as that estimate.
+ name: "failed_sample_of_an_estimated_shape_warns",
+ statements: []estimate{{"UPDATE t SET c = 1;", 0, errors.New("deadlock detected")}, {"UPDATE t SET c = 2;", 5, nil}},
+ dmlCount: 2,
+ wantRows: 10,
+ wantWarning: "Affected rows could not be estimated for 1 of 2 DML statements: deadlock detected",
+ wantExplains: 2,
+ },
+ {
+ // One sampled shape fails and two shapes are never sampled; all three count as the average of the other nine.
+ name: "failures_and_unsampled_shapes_warn_together",
+ statements: append(append([]estimate{{"DELETE FROM missing;", 0, errors.New("table missing does not exist")}}, repeat(9, "UPDATE t%d SET c = 1;", 10)...), estimate{"DELETE FROM a;", 1000, nil}, estimate{"DELETE FROM b;", 1000, nil}),
+ dmlCount: 12,
+ wantRows: 120,
+ wantWarning: "Affected rows could not be estimated for 3 of 12 DML statements: table missing does not exist",
+ wantExplains: 10,
+ },
+ {
+ name: "every_sample_failed",
+ statements: []estimate{{"d1", 0, errors.New("syntax error")}},
+ dmlCount: 1,
+ insertCount: 4,
+ wantRows: 4,
+ wantWarning: "Affected rows could not be estimated for 1 of 1 DML statements: syntax error",
+ wantExplains: 1,
+ },
+ {
+ name: "dml_without_text",
+ dmlCount: 2,
+ wantWarning: "Affected rows could not be estimated for 2 of 2 DML statements.",
+ },
+ {
+ name: "saturates_instead_of_overflowing",
+ statements: []estimate{{"u1", math.MaxInt64/2 + 1, nil}, {"u2", math.MaxInt64/2 + 1, nil}},
+ dmlCount: 3,
+ wantRows: math.MaxInt64,
+ wantWarning: "Affected rows could not be estimated for 1 of 3 DML statements.",
+ wantExplains: 2,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ changeSummary := &parserbase.ChangeSummary{
+ ChangedResources: model.NewChangedResources(nil /* dbMetadata */),
+ DMLCount: tc.dmlCount,
+ InsertCount: tc.insertCount,
+ }
+ estimates := map[string]estimate{}
+ for _, e := range tc.statements {
+ changeSummary.DMLStatements = append(changeSummary.DMLStatements, e.statement)
+ estimates[e.statement] = e
+ }
+ explains := 0
+ explain := func(_ context.Context, statement string) (int64, error) {
+ explains++
+ e := estimates[statement]
+ return e.rows, e.err
+ }
+
+ rows, warning := calculateAffectedRows(context.Background(), storepb.Engine_POSTGRES, changeSummary, explain)
+ require.Equal(t, tc.wantRows, rows)
+ require.Equal(t, tc.wantWarning, warning)
+ require.Equal(t, tc.wantExplains, explains)
+ })
+ }
+}
+
+func TestShapeKey(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statements []string
+ mysqlFamily bool
+ want string
+ }{
+ {
+ name: "literals and spacing",
+ statements: []string{
+ "UPDATE t SET v = 'it''s' WHERE id = 1;",
+ "UPDATE t SET v='x' WHERE id=20.5;",
+ "UPDATE t\n\tSET v = 'C:\\' WHERE id = 3;",
+ },
+ want: "UPDATE t SET v=? WHERE id=?;",
+ },
+ {
+ name: "MySQL family backslash escapes",
+ statements: []string{
+ "UPDATE t SET v = 'it\\'s' WHERE id = 1;",
+ "UPDATE t SET v = 'C:\\\\' WHERE id = 2;",
+ },
+ mysqlFamily: true,
+ want: "UPDATE t SET v=? WHERE id=?;",
+ },
+ {
+ name: "PostgreSQL escape strings",
+ statements: []string{
+ "UPDATE t SET v = E'it\\'s' WHERE id = 1;",
+ "UPDATE t SET v = E'x' WHERE id = 2;",
+ },
+ want: "UPDATE t SET v=E? WHERE id=?;",
+ },
+ {
+ name: "CockroachDB byte strings",
+ statements: []string{
+ "UPDATE t SET v = b'it\\'s' WHERE id = 1;",
+ "UPDATE t SET v = b'x' WHERE id = 2;",
+ },
+ want: "UPDATE t SET v=b? WHERE id=?;",
+ },
+ {
+ name: "ordinary comments",
+ statements: []string{
+ "-- row 1\nUPDATE t SET v = 1 WHERE id = 1;",
+ "/* row 2 */ UPDATE t SET v = 2 /* it's */ WHERE id = 2; -- done",
+ "UPDATE/**/t SET v = 3 WHERE id = 3;",
+ },
+ want: "UPDATE t SET v=? WHERE id=?;",
+ },
+ {
+ name: "MySQL family # comments",
+ statements: []string{
+ "# row 1\nUPDATE t SET v = 1 WHERE id = 1;",
+ "UPDATE t SET v = 2 WHERE id = 2; # row 2",
+ },
+ mysqlFamily: true,
+ want: "UPDATE t SET v=? WHERE id=?;",
+ },
+ {
+ name: "MySQL family -- without whitespace is not a comment",
+ statements: []string{
+ "UPDATE t SET c = c--1 WHERE id = 1;",
+ "UPDATE t SET c = c--2 WHERE id = 20;",
+ },
+ mysqlFamily: true,
+ want: "UPDATE t SET c=c--? WHERE id=?;",
+ },
+ {
+ name: "lists of literals",
+ statements: []string{
+ "DELETE FROM t WHERE id IN (1, 2, 3)",
+ "DELETE FROM t WHERE id IN ('a', 'b', 'c')",
+ },
+ want: "DELETE FROM t WHERE id IN(?,?,?)",
+ },
+ {
+ name: "numbers",
+ statements: []string{"UPDATE t1 SET c2 = 0x1F WHERE t1.c3 > 1e3", "UPDATE t1 SET c2 = 7 WHERE t1.c3 > 1.5e-3", "UPDATE t1 SET c2 = 0B1010 WHERE t1.c3 > 0o17", "UPDATE t1 SET c2 = 0xFF_FF WHERE t1.c3 > 1_000.5"},
+ want: "UPDATE t1 SET c2=? WHERE t1.c3>?",
+ },
+ {
+ name: "leading-dot decimals",
+ statements: []string{"UPDATE t SET v = .1 WHERE id > .5", "UPDATE t SET v = .10 WHERE id > .75e2"},
+ want: "UPDATE t SET v=.? WHERE id>.?",
+ },
+ {
+ name: "MySQL family numbers",
+ statements: []string{"UPDATE t1 SET c2 = 0x1F WHERE t1.c3 > 1e3", "UPDATE t1 SET c2 = 0b101 WHERE t1.c3 > 7"},
+ mysqlFamily: true,
+ want: "UPDATE t1 SET c2=? WHERE t1.c3>?",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ for _, statement := range tc.statements {
+ require.Equal(t, tc.want, shapeKey(statement, tc.mysqlFamily), statement)
+ }
+ })
+ }
+
+ t.Run("statements that differ beyond literals keep separate shapes", func(t *testing.T) {
+ for _, pair := range [][2]string{
+ {`UPDATE "Orders" SET v = 1`, `UPDATE "orders" SET v = 1`},
+ {`UPDATE "2024_orders" SET v = 1`, `UPDATE "2025_orders" SET v = 1`},
+ {"UPDATE `Orders` SET v = 1", "UPDATE `orders` SET v = 1"},
+ {"UPDATE Orders SET v = 1", "UPDATE orders SET v = 1"},
+ {"UPDATE [2024_orders] SET v = 1", "UPDATE [2025_orders] SET v = 1"},
+ {"UPDATE [1] SET v = 1", "UPDATE [2] SET v = 1"},
+ {"UPDATE [t]]1] SET v = 1", "UPDATE [t]]2] SET v = 1"},
+ {"UPDATE 2024_orders SET v = 1", "UPDATE 2025_orders SET v = 1"},
+ {"DELETE FROM t WHERE id IN (1)", "DELETE FROM t WHERE id IN (1, 2)"},
+ {"UPDATE t SET v = 1 /*!80000 WHERE id = 1 */", "UPDATE t SET v = 1"},
+ {"UPDATE /*+ CARDINALITY(t 1) */ t SET v = 1", "UPDATE /*+ CARDINALITY(t 1000000) */ t SET v = 1"},
+ {"UPDATE t SET v = 1 /*!80000 WHERE id = 1 */", "UPDATE t SET v = 1 /*!99999 WHERE id = 1 */"},
+ {"UPDATE t SET v = 1 /*M!100100 WHERE id = 1 */", "UPDATE t SET v = 1 /*M!999999 WHERE id = 1 */"},
+ {"UPDATE t SET v = 1 /*T![feature] WHERE id = 1 */", "UPDATE t SET v = 1"},
+ {"UPDATE --+ INDEX(t i1)\nt SET v = 1", "UPDATE --+ INDEX(t i2)\nt SET v = 1"},
+ {"UPDATE /*+ CARDINALITY(t 1000000) */ t SET v = 1", "UPDATE t SET v = 1"},
+ {"UPDATE [t--x] SET v = 1 WHERE id = 1", "UPDATE [t--x] SET v = 1"},
+ {"UPDATE t SET v = $$--$$ WHERE id = 1", "UPDATE t SET v = $$--$$"},
+ {"UPDATE t SET v = 1 -- don't\nWHERE id = 1", "UPDATE t SET v = 1 -- don't\n"},
+ {"UPDATE t SET v = v - -1 WHERE flag", "UPDATE t SET v = v --? WHERE flag"},
+ {"UPDATE t SET v = 1 -- x\nWHERE id = 1", "UPDATE t SET v = 1 -- x WHERE id=?"},
+ {"UPDATE t SET note = q'{it's}' WHERE id = 1 -- don't", "UPDATE t SET note = q'{it's}' WHERE id > 0 -- don't"},
+ {"UPDATE t SET v = 1 /* a /* b */ it's */ WHERE id = 1 -- don't", "UPDATE t SET v = 1 /* a /* b */ it's */ WHERE id > 0 -- don't"},
+ {"UPDATE t SET a = ARRAY[']'] WHERE id = 1 AND note <> 'a--b'", "UPDATE t SET a = ARRAY[']'] WHERE true AND note <> 'a--b'"},
+ // The quote never closes, so each statement keeps its own shape.
+ {"UPDATE t SET v = $$it's$$ WHERE id = 1", "UPDATE t SET v = $$it's$$"},
+ } {
+ require.NotEqual(t, shapeKey(pair[0], false), shapeKey(pair[1], false), pair[0])
+ require.NotEqual(t, shapeKey(pair[0], true), shapeKey(pair[1], true), pair[0])
+ }
+ // On MySQL, a word such as 0X1F, 0o17, or 1_000 is an identifier.
+ for _, pair := range [][2]string{{"UPDATE t SET v = 0X1F", "UPDATE t SET v = 0X20"}, {"UPDATE t SET v = 0o17", "UPDATE t SET v = 0o20"}, {"UPDATE t SET v = 1_000", "UPDATE t SET v = 1_001"}} {
+ require.NotEqual(t, shapeKey(pair[0], true), shapeKey(pair[1], true), pair[0])
+ }
+ // -- starts a comment on PostgreSQL, but on MySQL only before whitespace.
+ require.Equal(t, shapeKey("UPDATE t SET v = v--1 WHERE id = 1", false), shapeKey("UPDATE t SET v = v", false))
+ require.NotEqual(t, shapeKey("UPDATE t SET v = v--1 WHERE id = 1", true), shapeKey("UPDATE t SET v = v", true))
+ require.NotEqual(t, shapeKey("UPDATE t SET v = 'it\\'s' WHERE id = 1", true), shapeKey("UPDATE t SET v = 'it\\'s'", true))
+ require.NotEqual(t, shapeKey(`UPDATE t SET v = "a\"b" WHERE id = 1`, true), shapeKey(`UPDATE t SET v = "a\"b"`, true))
+ require.NotEqual(t, shapeKey("UPDATE t SET v = 1 # x\nWHERE id = 1", true), shapeKey("UPDATE t SET v = 1 # x WHERE id=?", true))
+ require.NotEqual(t, shapeKey("UPDATE t SET note = $$it's$$ WHERE id = 1 -- don't", false), shapeKey("UPDATE t SET note = $$it's$$ WHERE id > 0 -- don't", false))
+ require.NotEqual(t, shapeKey("UPDATE t SET note = $x$it's$x$ WHERE id = 1 -- don't", false), shapeKey("UPDATE t SET note = $x$it's$x$ WHERE id > 0 -- don't", false))
+ })
+}
+
// reportEngines derives the list from common.EngineSupportStatementReport so a
// newly admitted engine is covered without editing this test.
func reportEngines() []storepb.Engine {
diff --git a/backend/runner/plancheck/test/mysql_statement_type_check.yaml b/backend/runner/plancheck/test/mysql_statement_type_check.yaml
deleted file mode 100644
index 7bc3638fee..0000000000
--- a/backend/runner/plancheck/test/mysql_statement_type_check.yaml
+++ /dev/null
@@ -1,234 +0,0 @@
-- statement: |-
- UPDATE items,
- (SELECT id FROM items
- WHERE id IN
- (SELECT id FROM items
- WHERE retail / wholesale >= 1.3 AND quantity < 100))
- AS discounted
- SET items.retail = items.retail * 0.9
- WHERE items.id = discounted.id;
- changetype: 2
- want: []
-- statement: REPLACE INTO test2 VALUES (1, 'Old', '2014-08-20 18:47:00');
- changetype: 2
- want: []
-- statement: |-
- (SELECT 1 AS result UNION SELECT 2)
- ORDER BY result DESC LIMIT 1;
- changetype: 2
- want: []
-- statement: |-
- INSERT INTO tbl_temp2 (fld_id)
- SELECT tbl_temp1.fld_order_id
- FROM tbl_temp1 WHERE tbl_temp1.fld_order_id > 100;
- changetype: 1
- want: []
-- statement: IMPORT TABLE FROM '/tmp/mysql-files/*.sdi';
- changetype: 1
- want: []
-- statement: TABLE a EXCEPT TABLE b;
- changetype: 1
- want: []
-- statement: EXPLAIN SELECT c->>"$.name" AS name FROM jempn WHERE g > 2;
- changetype: 1
- want: []
-- statement: |-
- CREATE TABLE artists_and_works
- SELECT artist.name, COUNT(work.artist_id) AS number_of_works
- FROM artist LEFT JOIN work ON artist.id = work.artist_id
- GROUP BY artist.id;
- changetype: 1
- want: []
-- statement: |-
- CREATE EVENT e
- ON SCHEDULE
- EVERY 5 SECOND
- DO
- BEGIN
- DECLARE v INTEGER;
- DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
-
- SET v = 0;
-
- WHILE v < 5 DO
- INSERT INTO t1 VALUES (0);
- UPDATE t2 SET s1 = s1 + 1;
- SET v = v + 1;
- END WHILE;
- END;
- changetype: 1
- want: []
-- statement: |-
- CREATE TABLE customers (
- id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
- modified DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
- custinfo JSON,
- INDEX zips( (CAST(custinfo->'$.zipcode' AS UNSIGNED ARRAY)) )
- );
- changetype: 1
- want: []
-- statement: |-
- CREATE TABLE t1 (c1 INT);
- ALTER TABLE t1 ADD COLUMN c2 INT GENERATED ALWAYS AS (c1 + 1) STORED;
- changetype: 1
- want: []
-- statement: |-
- CREATE TABLE t1 (
- id INT,
- year_col INT
- )
- PARTITION BY RANGE (year_col) (
- PARTITION p0 VALUES LESS THAN (1991),
- PARTITION p1 VALUES LESS THAN (1995),
- PARTITION p2 VALUES LESS THAN (1999)
- );
- changetype: 1
- want: []
-- statement: ALTER SERVER s OPTIONS (USER 'sally');
- changetype: 1
- want: []
-- statement: |-
- ALTER LOGFILE GROUP lg_3
- ADD UNDOFILE 'undo_10.dat'
- INITIAL_SIZE=32M
- ENGINE=NDBCLUSTER;
- changetype: 1
- want: []
-- statement: ALTER FUNCTION hello COMMENT 'hello function';
- changetype: 2
- want: []
-- statement: |-
- CREATE DEFINER = 'admin'@'localhost' PROCEDURE account_count()
- BEGIN
- SELECT 'Number of accounts:', COUNT(*) FROM mysql.user;
- END;
- changetype: 2
- want: []
-- statement: ALTER PROCEDURE account_count COMMENT 'this account_count';
- changetype: 1
- want: []
-- statement: |-
- CREATE FUNCTION hello (s CHAR(20))
- RETURNS CHAR(50) DETERMINISTIC
- RETURN CONCAT('Hello, ',s,'!');
-
- SELECT hello('world');
- changetype: 2
- want: []
-- statement: |-
- delimiter //
- CREATE PROCEDURE citycount (IN country CHAR(3), OUT cities INT)
- BEGIN
- SELECT COUNT(*) INTO cities FROM world.city
- WHERE CountryCode = country;
- END//
- delimiter ;
- changetype: 2
- want: []
-- statement: CALL citycount('JPN', @cities); -- cities in Japan
- changetype: 2
- want: []
-- statement: ALTER EVENT no_such_event ON SCHEDULE EVERY '2:3' DAY_HOUR;
- changetype: 2
- want: []
-- statement: |-
- CREATE EVENT myevent
- ON SCHEDULE
- EVERY 6 HOUR
- COMMENT 'A sample comment.'
- DO
- UPDATE myschema.mytable SET mycol = mycol + 1;
- changetype: 2
- want: []
-- statement: ALTER DATABASE bbdev READ ONLY = DEFAULT;
- changetype: 2
- want: []
-- statement: ALTER SCHEMA bbdev READ ONLY = DEFAULT;
- changetype: 2
- want: []
-- statement: CREATE TABLE t1(a INT);
- changetype: 1
- want: []
-- statement: DELETE FROM t1;
- changetype: 1
- want: []
-- statement: SELECT 1;
- changetype: 2
- want: []
-- statement: DROP TABLE t1;
- changetype: 2
- want: []
-- statement: CREATE DATABASE bbdev;
- changetype: 1
- want:
- - |-
- {
- "status": "ERROR",
- "title": "Cannot create database",
- "content": "The statement \"CREATE DATABASE bbdev;\" creates database",
- "code": 404
- }
-- statement: DROP DATABASE bbdev;
- changetype: 2
- want:
- - |-
- {
- "status": "ERROR",
- "title": "Cannot drop database",
- "content": "The statement \"DROP DATABASE bbdev;\" drops database",
- "code": 403
- }
-- statement: RENAME TABLE t1 To t2;
- changetype: 2
- want: []
-- statement: TRUNCATE TABLE t1;
- changetype: 2
- want: []
-- statement: ALTER TABLE t1 ADD COLUMN col1 INT;
- changetype: 2
- want: []
-- statement: DELETE FROM t1 WHERE id = 1;
- changetype: 1
- want: []
-- statement: INSERT INTO t1(id) VALUE(1);
- changetype: 1
- want: []
-- statement: UPDATE t1 SET name = 'zp' WHERE id = 1;
- changetype: 1
- want: []
-- statement: |-
- update t_ci_job_artifact_config c
- inner join (
- select
- structure.id as structure_id,
- structure.job_id as job_id
- from
- t_ci_job_stage_group_structure structure,
- t_ci_stage_template_group stg
- where
- structure.stage_template_group_id = stg.id
- and stg.template_type = 3
- and stg.artifact_type in (4,5,6,7)
- ) tmp on c.job_id = tmp.job_id and c.type in (4,5,6,7)
- set c.structure_id = tmp.structure_id
- where c.structure_id is null;
- changetype: 1
- want: []
-- statement: |-
- update t_ci_job_artifact_config c
- inner join (
- select
- structure.id as structure_id,
- structure.job_id as job_id
- from
- t_ci_job_stage_group_structure structure,
- t_ci_stage_template_group stg
- where
- structure.stage_template_group_id = stg.id
- and stg.template_type = 3
- and stg.artifact_type in (4,5,6,7)
- ) tmp on c.job_id = tmp.job_id and c.type in (4,5,6,7)
- set c.structure_id = tmp.structure_id
- where c.structure_id is null;
- changetype: 2
- want: []
diff --git a/backend/runner/plancheck/test/test_affected_rows.yaml b/backend/runner/plancheck/test/test_affected_rows.yaml
deleted file mode 100644
index b61bd274f8..0000000000
--- a/backend/runner/plancheck/test/test_affected_rows.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-- statement: INSERT INTO t1 VALUES(1), (2), (3);
- want: 3
-- statement: ALTER TABLE t1 ADD COLUMN c2 INT;
- want: 100
-- statement: DROP TABLE t2;
- want: 1000
diff --git a/backend/store/model/changed_resources.go b/backend/store/model/changed_resources.go
index c61c7ef2f0..6407b36100 100644
--- a/backend/store/model/changed_resources.go
+++ b/backend/store/model/changed_resources.go
@@ -2,6 +2,9 @@ package model
import (
"slices"
+ "strings"
+
+ metadatapb "github.com/bytebase/omni/metadata"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
@@ -47,15 +50,7 @@ func (r *ChangedResources) Build() *storepb.ChangedResources {
d.Name = name
for _, schema := range d.Schemas {
for _, table := range schema.Tables {
- if r.dbMetadata == nil {
- continue
- }
- schemaMetadata := r.dbMetadata.GetSchemaMetadata(schema.GetName())
- if schemaMetadata == nil {
- continue
- }
- tableMetadata := schemaMetadata.GetTable(table.GetName())
- if tableMetadata != nil {
+ if tableMetadata := r.getTableMetadata(name, schema.GetName(), table.GetName()); tableMetadata != nil {
table.TableRows = tableMetadata.GetProto().GetRowCount()
}
}
@@ -65,6 +60,25 @@ func (r *ChangedResources) Build() *storepb.ChangedResources {
return changedResources
}
+// getTableMetadata returns the synced metadata of a changed table. The metadata only
+// describes the reviewed database, so a table qualified with another database has none,
+// even when the reviewed database has a table of the same name.
+func (r *ChangedResources) getTableMetadata(database, schema, table string) *TableMetadata {
+ if r.dbMetadata == nil {
+ return nil
+ }
+ if name := r.dbMetadata.DatabaseName(); name != "" {
+ if r.dbMetadata.GetIsObjectCaseSensitive() {
+ if database != name {
+ return nil
+ }
+ } else if !strings.EqualFold(database, name) {
+ return nil
+ }
+ }
+ return r.dbMetadata.GetSchemaMetadata(schema).GetTable(table)
+}
+
func (d *ChangedDatabase) build() *storepb.ChangedResourceDatabase {
changedResourceDatabase := &storepb.ChangedResourceDatabase{}
for name, schema := range d.schemas {
@@ -148,29 +162,22 @@ func (r *ChangedResources) GetDatabaseOnlyTargets() []string {
return result
}
+// CountAffectedTableRows sums the synced row counts of the tables changed by DDL. Partitions
+// have no row count of their own and resolve to their table, which is counted once.
func (r *ChangedResources) CountAffectedTableRows() int64 {
- if r.dbMetadata == nil {
- return 0
- }
-
+ counted := make(map[*metadatapb.TableMetadata]bool)
var totalAffectedRows int64
- for _, d := range r.databases {
+ for databaseName, d := range r.databases {
for schemaName, schema := range d.schemas {
for tableName, table := range schema.tables {
if !table.affectedTable {
continue
}
- if r.dbMetadata == nil {
- continue
- }
- schemaMeta := r.dbMetadata.GetSchemaMetadata(schemaName)
- if schemaMeta == nil {
- continue
- }
- tableMeta := schemaMeta.GetTable(tableName)
- if tableMeta == nil {
+ tableMeta := r.getTableMetadata(databaseName, schemaName, tableName)
+ if tableMeta == nil || counted[tableMeta.GetProto()] {
continue
}
+ counted[tableMeta.GetProto()] = true
totalAffectedRows += tableMeta.GetProto().GetRowCount()
}
}
diff --git a/backend/store/model/changed_resources_test.go b/backend/store/model/changed_resources_test.go
new file mode 100644
index 0000000000..84a9bc37ea
--- /dev/null
+++ b/backend/store/model/changed_resources_test.go
@@ -0,0 +1,102 @@
+package model
+
+import (
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+func TestCountAffectedTableRows(t *testing.T) {
+ type change struct {
+ database, schema, table string
+ affected bool
+ }
+ for _, tc := range []struct {
+ name string
+ engine storepb.Engine
+ changes []change
+ want int64
+ }{
+ {
+ name: "reviewed_database_table",
+ engine: storepb.Engine_POSTGRES,
+ changes: []change{{"db", "public", "t", true}},
+ want: 1_000_000,
+ },
+ {
+ name: "same_named_table_in_another_database",
+ engine: storepb.Engine_POSTGRES,
+ changes: []change{{"other_db", "public", "t", true}},
+ want: 0,
+ },
+ {
+ name: "table_changed_without_ddl",
+ engine: storepb.Engine_POSTGRES,
+ changes: []change{{"db", "public", "t", false}},
+ want: 0,
+ },
+ {
+ name: "partitions_count_their_table_once",
+ // The metadata has no per-partition row counts.
+ engine: storepb.Engine_POSTGRES,
+ changes: []change{
+ {"db", "public", "events", true},
+ {"db", "public", "events_2020", true},
+ {"db", "public", "events_2021", true},
+ },
+ want: 50_000_000,
+ },
+ {
+ name: "case_insensitive_database_name",
+ engine: storepb.Engine_MYSQL,
+ changes: []change{{"DB", "public", "t", true}},
+ want: 1_000_000,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ changedResources := NewChangedResources(newChangedResourcesTestMetadata(tc.engine))
+ for _, c := range tc.changes {
+ changedResources.AddTable(c.database, c.schema, &storepb.ChangedResourceTable{Name: c.table}, c.affected)
+ }
+ require.Equal(t, tc.want, changedResources.CountAffectedTableRows())
+ })
+ }
+}
+
+func TestBuildTableRows(t *testing.T) {
+ changedResources := NewChangedResources(newChangedResourcesTestMetadata(storepb.Engine_POSTGRES))
+ changedResources.AddTable("db", "public", &storepb.ChangedResourceTable{Name: "t"}, false)
+ changedResources.AddTable("other_db", "public", &storepb.ChangedResourceTable{Name: "t"}, false)
+
+ tableRows := map[string]int64{}
+ for _, database := range changedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ tableRows[database.GetName()+"."+table.GetName()] = table.GetTableRows()
+ }
+ }
+ }
+ require.Equal(t, map[string]int64{"db.t": 1_000_000, "other_db.t": 0}, tableRows)
+}
+
+// newChangedResourcesTestMetadata is the reviewed database "db". MySQL metadata is built
+// case-insensitive, as on a server with lower_case_table_names set.
+func newChangedResourcesTestMetadata(engine storepb.Engine) *DatabaseMetadata {
+ return NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {Name: "t", RowCount: 1_000_000},
+ {
+ Name: "events",
+ RowCount: 50_000_000,
+ Partitions: []*metadatapb.TablePartitionMetadata{{Name: "events_2020"}, {Name: "events_2021"}},
+ },
+ },
+ }},
+ }, nil, nil, engine, engine != storepb.Engine_MYSQL)
+}
diff --git a/backend/tests/gitops_test.go b/backend/tests/gitops_test.go
index 5718af8a3e..c77e215078 100644
--- a/backend/tests/gitops_test.go
+++ b/backend/tests/gitops_test.go
@@ -98,6 +98,12 @@ func TestGitOpsCheck(t *testing.T) {
Version: "002",
Statement: []byte(`CREATE INDEX idx_users_email ON users(email);`),
},
+ {
+ // Each file is checked against the database as it is, where users does not exist yet.
+ Path: "migrations/003__normalize_emails.sql",
+ Version: "003",
+ Statement: []byte(`UPDATE users SET email = lower(email);`),
+ },
},
}
@@ -112,17 +118,32 @@ func TestGitOpsCheck(t *testing.T) {
a.NoError(err)
a.NotNil(checkResp)
// The response contains results for each file-target combination
- a.Len(checkResp.Msg.Results, 2) // 2 files x 1 target = 2 results
+ a.Len(checkResp.Msg.Results, 3) // 3 files x 1 target = 3 results
// Verify check results.
targetCount := make(map[string]int)
+ updateChecked := false
for _, result := range checkResp.Msg.Results {
targetCount[result.Target]++
- // The check should complete successfully.
- a.NotNil(result)
+ var estimateWarnings []*v1pb.Advice
+ for _, advice := range result.Advices {
+ if advice.Title == "Affected rows estimate is incomplete" {
+ estimateWarnings = append(estimateWarnings, advice)
+ }
+ }
+ if result.File != "migrations/003__normalize_emails.sql" {
+ a.Empty(estimateWarnings, result.File)
+ continue
+ }
+ // The UPDATE cannot be explained before users exists, which the check reports instead of failing.
+ updateChecked = true
+ a.Len(estimateWarnings, 1)
+ a.Equal(v1pb.Advice_WARNING, estimateWarnings[0].Status)
+ a.Contains(estimateWarnings[0].Content, "Affected rows could not be estimated for 1 of 1 DML statements")
}
- // Should have 2 results for the single target (one for each file)
- a.Equal(2, targetCount[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])
+ a.True(updateChecked)
+ // Should have 3 results for the single target (one for each file)
+ a.Equal(3, targetCount[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])
// Test 2: Check release against multiple targets (test and prod).
checkRespMulti, err := ctl.releaseServiceClient.CheckRelease(ctx, connect.NewRequest(&v1pb.CheckReleaseRequest{
@@ -135,17 +156,17 @@ func TestGitOpsCheck(t *testing.T) {
}))
a.NoError(err)
a.NotNil(checkRespMulti)
- // 2 files x 2 targets = 4 results
- a.Len(checkRespMulti.Msg.Results, 4)
+ // 3 files x 2 targets = 6 results
+ a.Len(checkRespMulti.Msg.Results, 6)
// Verify both targets were checked.
checkedTargets := make(map[string]int)
for _, result := range checkRespMulti.Msg.Results {
checkedTargets[result.Target]++
}
- // Each target should have 2 results (one for each file)
- a.Equal(2, checkedTargets[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])
- a.Equal(2, checkedTargets[fmt.Sprintf("%s/databases/%s", prodInstance.Name, databaseName)])
+ // Each target should have 3 results (one for each file)
+ a.Equal(3, checkedTargets[fmt.Sprintf("%s/databases/%s", testInstance.Name, databaseName)])
+ a.Equal(3, checkedTargets[fmt.Sprintf("%s/databases/%s", prodInstance.Name, databaseName)])
}
func TestGitOpsCheckReleaseRiskLevel(t *testing.T) {
diff --git a/backend/tests/sql_dml_table_scope_test.go b/backend/tests/sql_dml_table_scope_test.go
index b921ca72d9..06f1bc830b 100644
--- a/backend/tests/sql_dml_table_scope_test.go
+++ b/backend/tests/sql_dml_table_scope_test.go
@@ -987,4 +987,50 @@ func TestSQLEditorTableScopedDMLEdgeCases(t *testing.T) {
resp, qErr := runAs(token, fmt.Sprintf("INSERT INTO %s.public.t VALUES (1);", otherProjDB))
assertDeniedOn(t, resp, qErr, fmt.Sprintf("databases/%s", otherProjDB))
})
+
+ // 18. A data-modifying CTE makes a SELECT a write: a SELECT-only user is denied, and a
+ // table-scoped DML grant authorizes it per CTE target.
+ t.Run("DataModifyingCTEAuthorizedAsDML", func(t *testing.T) {
+ ra := require.New(t)
+ email, token := newLimitedUser(t)
+ setProjectBindings(t, readBinding(email))
+
+ before := countRows(t, "public.t_src")
+ resp, qErr := runAs(token, "WITH d AS (DELETE FROM public.t_src RETURNING id) SELECT count(*) FROM d;")
+ assertDeniedOn(t, resp, qErr, "/tables/t_src")
+ ra.Equal(before, countRows(t, "public.t_src"), "the denied statement must not delete rows")
+
+ setProjectBindings(t, readBinding(email), tableScopedDML(email, "t_granted"))
+ before = countRows(t, "public.t_granted")
+ resp, qErr = runAs(token, "WITH i AS (INSERT INTO public.t_granted (id) VALUES (42) RETURNING id) SELECT id FROM i;")
+ assertAllowed(t, resp, qErr)
+ // The editor runs a write with Exec, so the rows the CTE returns never reach the response.
+ ra.Equal([]string{"Affected Rows"}, resp.Results[0].ColumnNames)
+ ra.Equal(before+1, countRows(t, "public.t_granted"))
+ resp, qErr = runAs(token, "WITH i AS (INSERT INTO public.t_other (id) VALUES (42) RETURNING id) SELECT count(*) FROM i;")
+ assertDeniedOn(t, resp, qErr, "/tables/t_other")
+ })
+
+ // 19. EXPLAIN ANALYZE executes the statement it explains, so explaining a write is authorized
+ // like the write, per target table.
+ t.Run("ExplainAnalyzeOfWriteAuthorizedAsDML", func(t *testing.T) {
+ ra := require.New(t)
+ email, token := newLimitedUser(t)
+ setProjectBindings(t, readBinding(email), tableScopedDML(email, "t_granted"))
+
+ before := countRows(t, "public.t_dst")
+ for _, statement := range []string{
+ "EXPLAIN ANALYZE MERGE INTO public.t_dst d USING public.t_src s ON d.id = s.id WHEN NOT MATCHED THEN INSERT (id) VALUES (s.id);",
+ "EXPLAIN ANALYZE WITH i AS (INSERT INTO public.t_dst (id) VALUES (1) RETURNING id) SELECT count(*) FROM i;",
+ } {
+ resp, qErr := runAs(token, statement)
+ assertDeniedOn(t, resp, qErr, "/tables/t_dst")
+ }
+ ra.Equal(before, countRows(t, "public.t_dst"), "the denied statements must not write rows")
+
+ before = countRows(t, "public.t_granted")
+ resp, qErr := runAs(token, "EXPLAIN ANALYZE WITH i AS (INSERT INTO public.t_granted (id) VALUES (43) RETURNING id) SELECT count(*) FROM i;")
+ assertAllowed(t, resp, qErr)
+ ra.Equal(before+1, countRows(t, "public.t_granted"))
+ })
}