Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(masking)!: refuse MySQL and MariaDB queries whose conditions read masked columns (#21712)

rebelice2450df0e7693
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.go
index e009bd276c..87c98755e0 100644
--- a/backend/api/v1/predicate_columns_check.go
+++ b/backend/api/v1/predicate_columns_check.go
@@ -147,5 +147,10 @@ func (m *maskingLevelEvaluator) isSensitiveColumn(database *store.DatabaseMessag
}
func isPredicateColumnsCheckEnabled(engine storepb.Engine) bool {
- return engine == storepb.Engine_MSSQL || engine == storepb.Engine_POSTGRES
+ switch engine {
+ case storepb.Engine_MSSQL, storepb.Engine_POSTGRES, storepb.Engine_MYSQL, storepb.Engine_MARIADB:
+ return true
+ default:
+ return false
+ }
}
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.go
index b523ba2943..bfe9a40805 100644
--- a/backend/common/testcontainer/shared.go
+++ b/backend/common/testcontainer/shared.go
@@ -25,6 +25,7 @@ var (
sharedTiDB = startOnce(getTiDBContainer)
sharedMongo = startOnce(getMongoDBContainer)
sharedOracle = startOnce(getOracleContainer)
+ sharedMySQL = startOnce(GetMySQLContainer)
sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))
// 10.7 is the oldest MariaDB release Bytebase supports.
sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))
@@ -113,6 +114,10 @@ func SharedMongoDBContainer(t testing.TB) *Container { return shared(t, sharedMo
// user is a DBA.
func SharedOracleContainer(t testing.TB) *Container { return shared(t, sharedOracle) }
+// SharedMySQLContainer is SharedPgContainer for MySQL 8.0; tests take a
+// database each with NewMySQLDatabase.
+func SharedMySQLContainer(t testing.TB) *Container { return shared(t, sharedMySQL) }
+
// SharedMariaDBContainer is SharedPgContainer for MariaDB 11.8; tests take a
// database each with NewMariaDBDatabase.
func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMariaDB) }
@@ -160,6 +165,13 @@ func NewTiDBDatabase(t testing.TB) (*Container, string) {
return createMySQLProtocolDatabase(t, SharedTiDBContainer(t))
}
+// NewMySQLDatabase creates an empty database for the test on the shared MySQL
+// container and returns the container with the database's name.
+func NewMySQLDatabase(t testing.TB) (*Container, string) {
+ t.Helper()
+ return createMySQLProtocolDatabase(t, SharedMySQLContainer(t))
+}
+
// NewMariaDBDatabase creates an empty database for the test on the shared
// MariaDB container and returns the container with the database's name.
func NewMariaDBDatabase(t testing.TB) (*Container, string) {
diff --git a/backend/plugin/parser/mysql/main_test.go b/backend/plugin/parser/mysql/main_test.go
new file mode 100644
index 0000000000..0a9c4993be
--- /dev/null
+++ b/backend/plugin/parser/mysql/main_test.go
@@ -0,0 +1,9 @@
+package mysql
+
+import (
+ "testing"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+)
+
+func TestMain(m *testing.M) { testcontainer.Main(m) }
diff --git a/backend/plugin/parser/mysql/query_span_extractor.go b/backend/plugin/parser/mysql/query_span_extractor.go
index ea7902703b..ca4a6a1326 100644
--- a/backend/plugin/parser/mysql/query_span_extractor.go
+++ b/backend/plugin/parser/mysql/query_span_extractor.go
@@ -2,6 +2,7 @@ package mysql
import (
"context"
+ "slices"
"strconv"
"strings"
@@ -25,8 +26,8 @@ type querySpanExtractor struct {
// ctes records common table expressions visible to the current select scope.
ctes []*base.PseudoTable
- // outerTableSources resolves correlated subquery column references.
- outerTableSources []base.TableSource
+ // outerScopes resolves correlated subquery column references, innermost last.
+ outerScopes []outerScope
// tableSourceFrom is the set of table sources visible from the current FROM clause.
tableSourceFrom []base.TableSource
@@ -34,6 +35,26 @@ type querySpanExtractor struct {
// priorTableInFrom resolves JSON_TABLE document expressions against preceding FROM items.
priorTableInFrom []base.TableSource
+ // clause, selectItems, groupBy, and inAggregate are the position in the
+ // current SELECT that decides how an unqualified name resolves.
+ clause clause
+ selectItems []base.QuerySpanResult
+ groupBy []ast.ExprNode
+ inAggregate bool
+ // inCondition and outputInCondition hide an EXISTS subquery's output,
+ // which decides nothing: inCondition within a condition of this SELECT,
+ // outputInCondition when this query's own output feeds a condition.
+ inCondition bool
+ outputInCondition bool
+
+ // predicateColumns is shared with the extractors of nested queries.
+ predicateColumns base.SourceColumnSet
+ // ctePredicateColumns holds each CTE's predicate columns until a query
+ // reads the CTE; MySQL never evaluates an unreferenced CTE.
+ ctePredicateColumns map[*base.PseudoTable]base.SourceColumnSet
+ // accessTables are the tables the statement reads, for the name fallback.
+ accessTables base.SourceColumnSet
+
viewResolutionStack map[string]bool
}
@@ -42,6 +63,8 @@ func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContext
defaultDatabase: defaultDatabase,
gCtx: gCtx,
ignoreCaseSensitive: ignoreCaseSensitive,
+ predicateColumns: make(base.SourceColumnSet),
+ ctePredicateColumns: make(map[*base.PseudoTable]base.SourceColumnSet),
}
}
@@ -63,127 +86,95 @@ const (
NaturalRightOuterJoin
)
+// joinTableSources returns the columns a join exposes. A column merged by
+// USING or NATURAL is COALESCE(left, right), and in matched rows both sides
+// hold its value, so it carries both sides' sources.
func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.TableSource {
switch tp {
- case Join, InnerJoin, CrossJoin, StraightJoin, LeftOuterJoin, RightOuterJoin:
- var columns []base.QuerySpanResult
- rightFieldsMap := make(map[string]bool)
- for _, field := range r.GetQuerySpanResult() {
- rightFieldsMap[strings.ToLower(field.Name)] = true
- }
- lowercaseUsingFields := make(map[string]bool)
- for _, field := range using {
- lowercaseUsingFields[strings.ToLower(field)] = true
- }
- for _, field := range l.GetQuerySpanResult() {
- columns = append(columns, field)
- if _, ok := lowercaseUsingFields[strings.ToLower(field.Name)]; ok {
- delete(rightFieldsMap, strings.ToLower(field.Name))
- }
- }
- for _, field := range r.GetQuerySpanResult() {
- if _, ok := rightFieldsMap[strings.ToLower(field.Name)]; ok {
- columns = append(columns, field)
+ case Join, InnerJoin, CrossJoin, StraightJoin, LeftOuterJoin, RightOuterJoin,
+ NaturalInnerJoin, NaturalLeftOuterJoin, NaturalRightOuterJoin:
+ default:
+ return nil
+ }
+ joined := make(map[string]bool)
+ for _, name := range joinedColumnNames(l, r, tp, using) {
+ joined[strings.ToLower(name)] = true
+ }
+ rightColumns := r.GetQuerySpanResult()
+ var columns []base.QuerySpanResult
+ for _, field := range l.GetQuerySpanResult() {
+ if joined[strings.ToLower(field.Name)] {
+ if rightField, ok := querySpanResultNamed(rightColumns, field.Name); ok {
+ field.SourceColumns, _ = base.MergeSourceColumnSet(field.SourceColumns, rightField.SourceColumns)
}
}
- return &base.PseudoTable{
- Columns: columns,
+ columns = append(columns, field)
+ }
+ for _, field := range rightColumns {
+ if !joined[strings.ToLower(field.Name)] {
+ columns = append(columns, field)
}
+ }
+ return &base.PseudoTable{
+ Columns: columns,
+ }
+}
+
+// joinedColumnNames returns the columns a join merges: the USING list, or for
+// a NATURAL join every column name both sides share.
+func joinedColumnNames(l, r base.TableSource, tp joinType, using []string) []string {
+ switch tp {
case NaturalInnerJoin, NaturalLeftOuterJoin, NaturalRightOuterJoin:
- rightFieldsMap := make(map[string]bool)
- for _, field := range r.GetQuerySpanResult() {
- rightFieldsMap[strings.ToLower(field.Name)] = true
- }
- var columns []base.QuerySpanResult
+ var names []string
+ rightColumns := r.GetQuerySpanResult()
for _, field := range l.GetQuerySpanResult() {
- columns = append(columns, field)
- delete(rightFieldsMap, strings.ToLower(field.Name))
- }
- for _, field := range r.GetQuerySpanResult() {
- if _, ok := rightFieldsMap[strings.ToLower(field.Name)]; ok {
- columns = append(columns, field)
+ if _, ok := querySpanResultNamed(rightColumns, field.Name); ok {
+ names = append(names, field.Name)
}
}
- return &base.PseudoTable{
- Columns: columns,
- }
+ return names
default:
- return nil
+ return using
}
}
-func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {
- findInTableSource := func(tableSource base.TableSource) ([]base.QuerySpanResult, bool) {
- if q.ignoreCaseSensitive {
- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {
- return nil, false
- }
- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {
- return nil, false
- }
- } else {
- if databaseName != "" && databaseName != tableSource.GetDatabaseName() {
- return nil, false
- }
- if tableName != "" && tableName != tableSource.GetTableName() {
- return nil, false
- }
+func querySpanResultNamed(results []base.QuerySpanResult, name string) (base.QuerySpanResult, bool) {
+ for _, result := range results {
+ if strings.EqualFold(result.Name, name) {
+ return result, true
}
- return tableSource.GetQuerySpanResult(), true
}
+ return base.QuerySpanResult{}, false
+}
- for i := len(q.outerTableSources) - 1; i >= 0; i-- {
- if querySpanResult, ok := findInTableSource(q.outerTableSources[i]); ok {
- return querySpanResult, true
- }
- }
- for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {
- if querySpanResult, ok := findInTableSource(q.tableSourceFrom[i]); ok {
- return querySpanResult, true
+func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {
+ for _, tables := range q.tableScopes() {
+ for i := len(tables) - 1; i >= 0; i-- {
+ if q.tableSourceMatches(tables[i], databaseName, tableName) {
+ return tables[i].GetQuerySpanResult(), true
+ }
}
}
return nil, false
}
+// getFieldColumnSource resolves a column reference as MySQL does: the current
+// SELECT first, by the rules of the clause the reference is in, then each
+// enclosing SELECT outward, its tables before the select list it exposes.
func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, error) {
databaseName = q.filterClusterName(databaseName)
- findInTableSource := func(tableSource base.TableSource) (base.SourceColumnSet, bool) {
- if q.ignoreCaseSensitive {
- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {
- return nil, false
- }
- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {
- return nil, false
- }
- } else {
- if databaseName != "" && databaseName != tableSource.GetDatabaseName() {
- return nil, false
- }
- if tableName != "" && tableName != tableSource.GetTableName() {
- return nil, false
- }
- }
- for _, column := range tableSource.GetQuerySpanResult() {
- if strings.EqualFold(column.Name, fieldName) {
- return column.SourceColumns, true
- }
- }
- return nil, false
+ if sourceColumnSet, ok := q.resolveInCurrentSelect(databaseName, tableName, fieldName); ok {
+ return sourceColumnSet, nil
}
-
- for i := len(q.outerTableSources) - 1; i >= 0; i-- {
- if sourceColumnSet, ok := findInTableSource(q.outerTableSources[i]); ok {
- return sourceColumnSet, nil
- }
- }
- for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {
- if sourceColumnSet, ok := findInTableSource(q.tableSourceFrom[i]); ok {
+ for i := len(q.outerScopes) - 1; i >= 0; i-- {
+ scope := q.outerScopes[i]
+ if sourceColumnSet, ok := q.findColumnInTables(scope.tables, databaseName, tableName, fieldName); ok {
return sourceColumnSet, nil
}
- }
- for i := len(q.priorTableInFrom) - 1; i >= 0; i-- {
- if sourceColumnSet, ok := findInTableSource(q.priorTableInFrom[i]); ok {
- return sourceColumnSet, nil
+ if databaseName == "" && tableName == "" {
+ if sourceColumnSet, ok := selectItemsNamed(scope.items, fieldName); ok {
+ return sourceColumnSet, nil
+ }
}
}
@@ -194,6 +185,50 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, field
}
}
+// tableScopes returns the table sources a name can resolve to, nearest first.
+func (q *querySpanExtractor) tableScopes() [][]base.TableSource {
+ scopes := [][]base.TableSource{q.tableSourceFrom, q.priorTableInFrom}
+ for i := len(q.outerScopes) - 1; i >= 0; i-- {
+ scopes = append(scopes, q.outerScopes[i].tables)
+ }
+ return scopes
+}
+
+func (q *querySpanExtractor) tableSourceMatches(tableSource base.TableSource, databaseName, tableName string) bool {
+ if q.ignoreCaseSensitive {
+ if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {
+ return false
+ }
+ if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {
+ return false
+ }
+ return true
+ }
+ if databaseName != "" && databaseName != tableSource.GetDatabaseName() {
+ return false
+ }
+ if tableName != "" && tableName != tableSource.GetTableName() {
+ return false
+ }
+ return true
+}
+
+// findColumnInTables searches tables last to first, so a join's merged
+// columns, appended after its operands, win for an unqualified name.
+func (q *querySpanExtractor) findColumnInTables(tables []base.TableSource, databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {
+ for i := len(tables) - 1; i >= 0; i-- {
+ if !q.tableSourceMatches(tables[i], databaseName, tableName) {
+ continue
+ }
+ for _, column := range tables[i].GetQuerySpanResult() {
+ if strings.EqualFold(column.Name, fieldName) {
+ return column.SourceColumns, true
+ }
+ }
+ }
+ return nil, false
+}
+
func (q *querySpanExtractor) filterClusterName(databaseName string) string {
if q.gCtx.Engine == storepb.Engine_STARROCKS {
list := strings.Split(databaseName, ":")
@@ -209,6 +244,7 @@ func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (ba
for i := len(q.ctes) - 1; i >= 0; i-- {
table := q.ctes[i]
if table.Name == tableName {
+ mergeSourceColumnSetInto(q.predicateColumns, q.ctePredicateColumns[table])
return table, nil
}
}
@@ -434,7 +470,9 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statement
explainAnalyze := isExplain && explain.Analyze
var results []base.QuerySpanResult
+ predicateColumns := base.SourceColumnSet{}
if queryType == base.Select {
+ q.accessTables = accessTables
tableSource, err := q.extractOmniSelectRoot(root)
if err != nil {
var resourceNotFound *base.ResourceNotFoundError
@@ -443,20 +481,21 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statement
Type: base.Select,
SourceColumns: accessTables,
Results: []base.QuerySpanResult{},
- PredicateColumns: base.SourceColumnSet{},
+ PredicateColumns: q.predicateColumns,
NotFoundError: resourceNotFound,
}, nil
}
return nil, err
}
results = tableSource.GetQuerySpanResult()
+ predicateColumns = q.predicateColumns
}
return &base.QuerySpan{
Type: queryType,
ExplainAnalyze: explainAnalyze,
SourceColumns: accessTables,
Results: results,
- PredicateColumns: base.SourceColumnSet{},
+ PredicateColumns: predicateColumns,
}, nil
}
@@ -465,6 +504,9 @@ func (q *omniQuerySpanExtractor) extractOmniSelectRoot(root ast.Node) (*base.Pse
case *ast.SelectStmt:
return q.extractOmniSelectStmt(n)
case *ast.ExplainStmt:
+ if n.Analyze {
+ q.collectExplainedQueryColumns(n.Stmt)
+ }
return &base.PseudoTable{}, nil
case *ast.TableStmt:
return q.extractOmniTableStmt(n)
@@ -486,38 +528,74 @@ func (q *omniQuerySpanExtractor) extractOmniSelectStmt(stmt *ast.SelectStmt) (*b
originalTableSourceFrom := q.tableSourceFrom
originalPriorLength := len(q.priorTableInFrom)
originalCTELength := len(q.ctes)
+ originalPosition := q.position()
defer func() {
q.tableSourceFrom = originalTableSourceFrom
q.priorTableInFrom = q.priorTableInFrom[:originalPriorLength]
q.ctes = q.ctes[:originalCTELength]
+ q.setPosition(originalPosition)
}()
+ q.setPosition(selectPosition{})
if err := q.processOmniCTEs(stmt.CTEs); err != nil {
return nil, err
}
- if inner := stmt.ParenSource; inner != nil {
- return q.extractOmniSelectStmt(inner)
- }
- if tableStmt := stmt.TableSource; tableStmt != nil {
- return q.extractOmniTableStmt(tableStmt)
- }
- if valuesStmt := stmt.ValuesSource; valuesStmt != nil {
- return q.extractOmniValuesStmt(valuesStmt)
+ var table *base.PseudoTable
+ var err error
+ switch {
+ case stmt.ParenSource != nil:
+ table, err = q.extractOmniSelectStmt(stmt.ParenSource)
+ case stmt.TableSource != nil:
+ table, err = q.extractOmniTableStmt(stmt.TableSource)
+ case stmt.ValuesSource != nil:
+ table, err = q.extractOmniValuesStmt(stmt.ValuesSource)
+ case stmt.SetOp != ast.SetOpNone:
+ table, err = q.extractOmniSetOp(stmt)
+ default:
+ return q.extractOmniQuerySpecification(stmt)
}
- if stmt.SetOp != ast.SetOpNone {
- return q.extractOmniSetOp(stmt)
+ if err != nil {
+ return nil, err
}
+ // The ORDER BY and LIMIT written after a parenthesized query, a TABLE or
+ // VALUES primary, or a set operation belong to the wrapper.
+ q.selectItems = table.GetQuerySpanResult()
+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)
+ return table, nil
+}
+// extractOmniQuerySpecification extracts a SELECT ... FROM ... block,
+// recording the columns its conditions read as it resolves each clause.
+func (q *omniQuerySpanExtractor) extractOmniQuerySpecification(stmt *ast.SelectStmt) (*base.PseudoTable, error) {
+ q.clause = clauseFrom
fromSources, err := q.extractOmniTableSources(stmt.From)
if err != nil {
return nil, err
}
q.tableSourceFrom = append(q.tableSourceFrom, fromSources...)
+ q.clause = clauseWhere
+ q.collectConditionColumns(stmt.Where)
+
+ q.clause = clauseSelectList
results, err := q.extractOmniTargetList(stmt.TargetList, fromSources)
if err != nil {
return nil, err
}
+ q.selectItems = results
+
+ q.groupBy = stmt.GroupBy
+ q.clause = clauseGroupBy
+ for _, expr := range stmt.GroupBy {
+ q.collectNestedQueryColumns(expr)
+ }
+ q.clause = clauseHaving
+ q.collectConditionColumns(stmt.Having)
+ q.clause = clauseOrderBy
+ for _, window := range stmt.WindowClause {
+ q.collectNestedQueryColumns(window)
+ }
+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)
return &base.PseudoTable{
Columns: results,
}, nil
@@ -534,21 +612,36 @@ func (q *omniQuerySpanExtractor) extractOmniTableStmt(stmt *ast.TableStmt) (*bas
if err != nil {
return nil, err
}
+ q.selectItems = tableSource.GetQuerySpanResult()
+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)
return &base.PseudoTable{Columns: tableSource.GetQuerySpanResult()}, nil
}
+// extractOmniValuesStmt merges every row into its column, since each row is
+// returned.
func (q *omniQuerySpanExtractor) extractOmniValuesStmt(stmt *ast.ValuesStmt) (*base.PseudoTable, error) {
if stmt == nil || len(stmt.Rows) == 0 {
return &base.PseudoTable{}, nil
}
var columns []base.QuerySpanResult
- for _, expr := range stmt.Rows[0] {
- field, err := q.extractOmniExpr(expr)
- if err != nil {
- return nil, err
+ for i, row := range stmt.Rows {
+ for j, expr := range row {
+ field, err := q.extractOmniExpr(expr)
+ if err != nil {
+ return nil, err
+ }
+ if i == 0 {
+ columns = append(columns, field)
+ continue
+ }
+ if j < len(columns) {
+ columns[j].SourceColumns, _ = base.MergeSourceColumnSet(columns[j].SourceColumns, field.SourceColumns)
+ columns[j].IsPlainField = columns[j].IsPlainField && field.IsPlainField
+ }
}
- columns = append(columns, field)
}
+ q.selectItems = columns
+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)
return &base.PseudoTable{Columns: columns}, nil
}
@@ -569,9 +662,15 @@ func (q *omniQuerySpanExtractor) extractOmniSetOp(stmt *ast.SelectStmt) (*base.P
if len(leftResults) != len(rightResults) {
return nil, errors.Errorf("MySQL UNION operator left has %d fields, right has %d fields", len(leftResults), len(rightResults))
}
+ // INTERSECT and EXCEPT keep a row by whether the other branch has it, so
+ // both branches' output decides which rows come back.
+ filters := stmt.SetOp == ast.SetOpIntersect || stmt.SetOp == ast.SetOpExcept
results := make([]base.QuerySpanResult, 0, len(leftResults))
for i := range leftResults {
sourceColumns, _ := base.MergeSourceColumnSet(leftResults[i].SourceColumns, rightResults[i].SourceColumns)
+ if filters {
+ mergeSourceColumnSetInto(q.predicateColumns, sourceColumns)
+ }
results = append(results, base.QuerySpanResult{
Name: leftResults[i].Name,
SourceColumns: sourceColumns,
@@ -581,13 +680,20 @@ func (q *omniQuerySpanExtractor) extractOmniSetOp(stmt *ast.SelectStmt) (*base.P
return &base.PseudoTable{Columns: results}, nil
}
+// processOmniCTEs extracts each CTE into its own predicate set, which
+// findTableSchema merges into the reading query's.
func (q *omniQuerySpanExtractor) processOmniCTEs(ctes []*ast.CommonTableExpr) error {
for _, cte := range ctes {
+ readerPredicateColumns := q.predicateColumns
+ q.predicateColumns = make(base.SourceColumnSet)
tableSource, err := q.extractOmniCTE(cte)
+ ctePredicateColumns := q.predicateColumns
+ q.predicateColumns = readerPredicateColumns
if err != nil {
return err
}
if tableSource != nil {
+ q.ctePredicateColumns[tableSource] = ctePredicateColumns
q.ctes = append(q.ctes, tableSource)
}
}
@@ -743,6 +849,8 @@ func (q *omniQuerySpanExtractor) extractOmniJSONTable(expr *ast.JsonTableExpr) (
if err != nil {
return nil, err
}
+ // JSON_TABLE returns a row per match in the document.
+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)
var columns []base.QuerySpanResult
for _, column := range flattenOmniJSONTableColumns(expr.Columns) {
columns = append(columns, base.QuerySpanResult{
@@ -789,10 +897,26 @@ func (q *omniQuerySpanExtractor) extractOmniJoin(join *ast.JoinClause) (base.Tab
}
var using []string
- if condition, ok := join.Condition.(*ast.UsingCondition); ok && condition != nil {
- using = condition.Columns
+ switch condition := join.Condition.(type) {
+ case *ast.OnCondition:
+ if condition != nil {
+ q.collectConditionColumns(condition.Expr)
+ }
+ case *ast.UsingCondition:
+ if condition != nil {
+ using = condition.Columns
+ }
+ default:
+ }
+ tp := omniJoinType(join.Type)
+ if left != nil && right != nil {
+ // USING and NATURAL compare the joined columns of both sides.
+ for _, name := range joinedColumnNames(left, right, tp, using) {
+ q.recordJoinedColumn(left, name)
+ q.recordJoinedColumn(right, name)
+ }
}
- return joinTableSources(left, right, omniJoinType(join.Type), using), nil
+ return joinTableSources(left, right, tp, using), nil
}
func omniJoinType(joinType ast.JoinType) joinType {
@@ -822,10 +946,14 @@ func (q *omniQuerySpanExtractor) extractOmniSubquery(expr *ast.SubqueryExpr) (*b
if expr == nil {
return &base.PseudoTable{}, nil
}
- subqueryExtractor := q.cloneOmniForSubquery()
+ // A derived table cannot see the FROM clause it sits in unless it is
+ // LATERAL, and then only the items before it: earlier FROM items, and the
+ // operands extractOmniJoin has already added for the join it is in.
+ scope := outerScope{}
if expr.Lateral {
- subqueryExtractor.outerTableSources = append(subqueryExtractor.outerTableSources, q.priorTableInFrom...)
+ scope.tables = slices.Concat(q.tableSourceFrom, q.priorTableInFrom)
}
+ subqueryExtractor := q.cloneOmniWithScope(scope)
tableSource, err := subqueryExtractor.extractOmniSelectStmt(expr.Select)
if err != nil {
return nil, err
@@ -848,16 +976,37 @@ func (q *omniQuerySpanExtractor) extractOmniSubquery(expr *ast.SubqueryExpr) (*b
}, nil
}
+// cloneOmniForSubquery returns the extractor for a query nested in an
+// expression of the current SELECT, which sees its tables and, where MySQL
+// allows, its select list.
func (q *omniQuerySpanExtractor) cloneOmniForSubquery() *omniQuerySpanExtractor {
+ return q.cloneOmniWithScope(outerScope{
+ tables: slices.Concat(q.tableSourceFrom, q.priorTableInFrom),
+ items: q.selectItemsVisibleToSubquery(),
+ })
+}
+
+// cloneOmniWithScope returns an extractor for a nested query that sees scope
+// as its nearest enclosing SELECT. The nested query records its predicate
+// columns into the same set.
+func (q *omniQuerySpanExtractor) cloneOmniWithScope(scope outerScope) *omniQuerySpanExtractor {
+ outerScopes := slices.Clone(q.outerScopes)
+ if len(scope.tables) > 0 || len(scope.items) > 0 {
+ outerScopes = append(outerScopes, scope)
+ }
return &omniQuerySpanExtractor{
querySpanExtractor: &querySpanExtractor{
ctx: q.ctx,
defaultDatabase: q.defaultDatabase,
gCtx: q.gCtx,
ctes: q.ctes,
- outerTableSources: append(q.outerTableSources, q.tableSourceFrom...),
+ outerScopes: outerScopes,
tableSourceFrom: []base.TableSource{},
ignoreCaseSensitive: q.ignoreCaseSensitive,
+ outputInCondition: q.inCondition || q.outputInCondition,
+ predicateColumns: q.predicateColumns,
+ ctePredicateColumns: q.ctePredicateColumns,
+ accessTables: q.accessTables,
viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),
},
source: q.source,
@@ -904,6 +1053,8 @@ func (q *omniQuerySpanExtractor) extractOmniTargetList(targets []ast.ExprNode, f
default:
}
+ // A subquery in this item can name the items before it.
+ q.selectItems = result
field, err := q.extractOmniTarget(target)
if err != nil {
return nil, err
@@ -999,7 +1150,13 @@ func (q *omniQuerySpanExtractor) extractOmniExpr(expr ast.ExprNode) (base.QueryS
}
}
}
+ // HAVING resolves the arguments of an aggregate like a GROUP BY name.
+ inAggregate := q.inAggregate
+ if q.clause == clauseHaving && e.Over == nil && isAggregateFunction(e.Name) {
+ q.inAggregate = true
+ }
sourceColumns, err := q.mergeOmniExprSources(exprs...)
+ q.inAggregate = inAggregate
if err != nil {
return base.QuerySpanResult{}, err
}
@@ -1015,6 +1172,10 @@ func (q *omniQuerySpanExtractor) extractOmniExpr(expr ast.ExprNode) (base.QueryS
if err != nil {
return base.QuerySpanResult{}, err
}
+ if q.inCondition || q.outputInCondition {
+ // EXISTS reports whether a row comes back, not what the row holds.
+ sourceColumns = base.SourceColumnSet{}
+ }
return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil
case *ast.CaseExpr:
var exprs []ast.ExprNode
diff --git a/backend/plugin/parser/mysql/query_span_extractor_test.go b/backend/plugin/parser/mysql/query_span_extractor_test.go
index 42267a4209..01f25204cd 100644
--- a/backend/plugin/parser/mysql/query_span_extractor_test.go
+++ b/backend/plugin/parser/mysql/query_span_extractor_test.go
@@ -94,8 +94,8 @@ func TestOmniQuerySpanScaffold_QueryTypesAndAccessTables(t *testing.T) {
require.Equal(t, sourceColumnSetFromResources(tc.wantResources), span.SourceColumns)
if tc.wantType != base.Select {
require.Empty(t, span.Results)
+ require.Empty(t, span.PredicateColumns)
}
- require.Empty(t, span.PredicateColumns)
})
}
}
@@ -818,12 +818,17 @@ func TestOmniQuerySpanScenarioBatch2_FromJoinAndScopeCoverage(t *testing.T) {
"SELECT * FROM t1 NATURAL LEFT JOIN t2",
"SELECT * FROM t1 NATURAL RIGHT JOIN t2",
}
+ // A NATURAL join merges each shared column, which holds both sides' value.
want := []base.QuerySpanResult{
+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},
+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}, {Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},
+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}, {Database: "db", Table: "t2", Column: "c"}}), IsPlainField: true},
+ }
+ wantNonNatural := []base.QuerySpanResult{
{Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},
{Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}}), IsPlainField: true},
{Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}}), IsPlainField: true},
}
- wantNonNatural := append([]base.QuerySpanResult{}, want...)
wantNonNatural = append(wantNonNatural,
base.QuerySpanResult{Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},
base.QuerySpanResult{Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},
@@ -1041,17 +1046,17 @@ func TestOmniQuerySpanScenarioBatch2_SubqueryCTEAndSetCoverage(t *testing.T) {
},
},
{
- name: "unqualified_inner_column_uses_legacy_resolution_order",
+ name: "unqualified_inner_column_resolves_in_the_subquery_first",
statement: "SELECT (SELECT a FROM t2) AS inner_a FROM t1",
want: []base.QuerySpanResult{
- {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},
+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},
},
},
{
- name: "inner_alias_shadow_keeps_legacy_outer_first_lookup",
+ name: "inner_alias_shadows_the_outer_alias",
statement: "SELECT (SELECT x.a FROM t2 AS x) AS inner_a FROM t1 AS x",
want: []base.QuerySpanResult{
- {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},
+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},
},
},
{
diff --git a/backend/plugin/parser/mysql/query_span_predicate.go b/backend/plugin/parser/mysql/query_span_predicate.go
new file mode 100644
index 0000000000..f4fd181d3d
--- /dev/null
+++ b/backend/plugin/parser/mysql/query_span_predicate.go
@@ -0,0 +1,296 @@
+package mysql
+
+import (
+ "strings"
+
+ "github.com/bytebase/omni/mysql/ast"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// clause is where a name sits in a SELECT, which decides what it can resolve
+// to. query_span_predicate_test.go checks each rule against MySQL and MariaDB.
+type clause int
+
+const (
+ // clauseSelectList, clauseFrom, and clauseWhere see the FROM clause only.
+ clauseSelectList clause = iota
+ clauseFrom
+ clauseWhere
+ // clauseGroupBy sees the FROM clause, then the select list.
+ clauseGroupBy
+ // clauseHaving sees a GROUP BY column, then the select list, then the FROM
+ // clause; inside an aggregate it resolves like clauseGroupBy.
+ clauseHaving
+ // clauseOrderBy resolves no names itself, since the ORDER BY keys are not
+ // predicates; queries nested in it see the select list.
+ clauseOrderBy
+)
+
+// outerScope is what an enclosing SELECT shows a query nested in it.
+type outerScope struct {
+ tables []base.TableSource
+ // items is the select list the nested query can name, set only where
+ // MySQL lets it: the select list (items before it), GROUP BY, HAVING, and
+ // ORDER BY.
+ items []base.QuerySpanResult
+}
+
+// selectPosition is the per-SELECT state a nested SELECT saves and restores.
+type selectPosition struct {
+ clause clause
+ selectItems []base.QuerySpanResult
+ groupBy []ast.ExprNode
+ inAggregate bool
+ inCondition bool
+}
+
+func (q *querySpanExtractor) position() selectPosition {
+ return selectPosition{
+ clause: q.clause,
+ selectItems: q.selectItems,
+ groupBy: q.groupBy,
+ inAggregate: q.inAggregate,
+ inCondition: q.inCondition,
+ }
+}
+
+func (q *querySpanExtractor) setPosition(p selectPosition) {
+ q.clause = p.clause
+ q.selectItems = p.selectItems
+ q.groupBy = p.groupBy
+ q.inAggregate = p.inAggregate
+ q.inCondition = p.inCondition
+}
+
+func (q *querySpanExtractor) selectItemsVisibleToSubquery() []base.QuerySpanResult {
+ switch q.clause {
+ case clauseSelectList, clauseGroupBy, clauseHaving, clauseOrderBy:
+ return q.selectItems
+ default:
+ return nil
+ }
+}
+
+// resolveInCurrentSelect resolves a name in the SELECT being extracted.
+func (q *querySpanExtractor) resolveInCurrentSelect(databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {
+ fromClause := func(databaseName, tableName string) (base.SourceColumnSet, bool) {
+ if columns, ok := q.findColumnInTables(q.tableSourceFrom, databaseName, tableName, fieldName); ok {
+ return columns, true
+ }
+ return q.findColumnInTables(q.priorTableInFrom, databaseName, tableName, fieldName)
+ }
+ if databaseName != "" || tableName != "" {
+ return fromClause(databaseName, tableName)
+ }
+ switch {
+ case q.clause == clauseHaving && !q.inAggregate:
+ // MySQL warns when a GROUP BY column and a select alias share the name
+ // and takes the column; MariaDB refuses the query.
+ if group := q.groupByColumnNamed(fieldName); group != nil {
+ if columns, ok := fromClause(group.Schema, group.Table); ok {
+ return columns, true
+ }
+ }
+ if columns, ok := selectItemsNamed(q.selectItems, fieldName); ok {
+ return columns, true
+ }
+ return fromClause("", "")
+ case q.clause == clauseHaving, q.clause == clauseGroupBy:
+ if columns, ok := fromClause("", ""); ok {
+ return columns, true
+ }
+ return selectItemsNamed(q.selectItems, fieldName)
+ default:
+ return fromClause("", "")
+ }
+}
+
+func (q *querySpanExtractor) groupByColumnNamed(name string) *ast.ColumnRef {
+ for _, expr := range q.groupBy {
+ if ref, ok := expr.(*ast.ColumnRef); ok && !ref.Star && strings.EqualFold(ref.Column, name) {
+ return ref
+ }
+ }
+ return nil
+}
+
+// selectItemsNamed returns the sources of every select item called name. Two
+// different items with one name make MySQL refuse the query.
+func selectItemsNamed(items []base.QuerySpanResult, name string) (base.SourceColumnSet, bool) {
+ columns := make(base.SourceColumnSet)
+ found := false
+ for _, item := range items {
+ if strings.EqualFold(item.Name, name) {
+ mergeSourceColumnSetInto(columns, item.SourceColumns)
+ found = true
+ }
+ }
+ return columns, found
+}
+
+// collectConditionColumns records the columns expr reads, where the database
+// evaluates it on unmasked values to decide which rows come back.
+func (q *omniQuerySpanExtractor) collectConditionColumns(expr ast.ExprNode) {
+ if expr == nil {
+ return
+ }
+ inCondition := q.inCondition
+ q.inCondition = true
+ field, err := q.extractOmniExpr(expr)
+ q.inCondition = inCondition
+ if err != nil {
+ q.recordColumnsNamedIn(expr)
+ return
+ }
+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)
+}
+
+// collectNestedQueryColumns records the predicate columns of the queries
+// nested in node, whose own value only groups or orders the rows.
+func (q *omniQuerySpanExtractor) collectNestedQueryColumns(node ast.Node) {
+ if node == nil {
+ return
+ }
+ ast.Inspect(node, func(n ast.Node) bool {
+ sel, ok := n.(*ast.SelectStmt)
+ if !ok {
+ return true
+ }
+ if _, _, err := q.extractOmniSelectSourceColumns(sel); err != nil {
+ q.recordColumnsNamedIn(sel)
+ }
+ return false
+ })
+}
+
+// collectTrailingClauses records what ORDER BY and LIMIT decide: the queries
+// nested in ORDER BY, and the row count. The ORDER BY keys only order rows.
+func (q *omniQuerySpanExtractor) collectTrailingClauses(orderBy []*ast.OrderByItem, limit *ast.Limit) {
+ q.clause = clauseOrderBy
+ for _, item := range orderBy {
+ if item != nil {
+ q.collectNestedQueryColumns(item.Expr)
+ }
+ }
+ if limit != nil {
+ q.collectConditionColumns(limit.Count)
+ q.collectConditionColumns(limit.Offset)
+ }
+}
+
+// collectExplainedQueryColumns records the predicate columns of the query
+// EXPLAIN ANALYZE runs, whose plan reports the rows each condition kept. The
+// plan is not a result set, so the query's lineage is dropped.
+func (q *omniQuerySpanExtractor) collectExplainedQueryColumns(stmt ast.StmtNode) {
+ if _, err := q.extractOmniSelectRoot(stmt); err != nil {
+ q.recordColumnsNamedIn(stmt)
+ }
+}
+
+func (q *omniQuerySpanExtractor) recordJoinedColumn(table base.TableSource, name string) {
+ if field, ok := querySpanResultNamed(table.GetQuerySpanResult(), name); ok {
+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)
+ return
+ }
+ q.recordColumnsNamed("", "", name)
+}
+
+// recordColumnsNamedIn over-approximates the columns node reads by name, for
+// a condition the extractor could not resolve.
+func (q *omniQuerySpanExtractor) recordColumnsNamedIn(node ast.Node) {
+ ast.Inspect(node, func(n ast.Node) bool {
+ if ref, ok := n.(*ast.ColumnRef); ok && !ref.Star {
+ q.recordColumnsNamed(ref.Schema, ref.Table, ref.Column)
+ }
+ return true
+ })
+}
+
+// recordColumnsNamed records every column called fieldName that the reference
+// could mean: in any visible table or select list, and in any table the
+// statement reads. A qualifier that names none of them is ignored.
+func (q *omniQuerySpanExtractor) recordColumnsNamed(databaseName, tableName, fieldName string) {
+ qualifierMatched := false
+ recordTables := func(tables []base.TableSource) {
+ for _, table := range tables {
+ if !q.tableSourceMatches(table, databaseName, tableName) {
+ continue
+ }
+ qualifierMatched = true
+ for _, column := range table.GetQuerySpanResult() {
+ if strings.EqualFold(column.Name, fieldName) {
+ mergeSourceColumnSetInto(q.predicateColumns, column.SourceColumns)
+ }
+ }
+ }
+ }
+ for _, tables := range q.tableScopes() {
+ recordTables(tables)
+ }
+ if databaseName == "" && tableName == "" {
+ columns, _ := selectItemsNamed(q.selectItems, fieldName)
+ mergeSourceColumnSetInto(q.predicateColumns, columns)
+ for _, scope := range q.outerScopes {
+ columns, _ := selectItemsNamed(scope.items, fieldName)
+ mergeSourceColumnSetInto(q.predicateColumns, columns)
+ }
+ }
+
+ for access := range q.accessTables {
+ if access.Table == "" {
+ continue
+ }
+ if tableName != "" && qualifierMatched && !q.tableSourceMatches(&base.PhysicalTable{Database: access.Database, Name: access.Table}, databaseName, tableName) {
+ continue
+ }
+ for _, column := range q.metadataColumnsNamed(access.Database, access.Table, fieldName) {
+ q.predicateColumns[base.ColumnResource{Database: access.Database, Table: access.Table, Column: column}] = true
+ }
+ }
+}
+
+func (q *omniQuerySpanExtractor) metadataColumnsNamed(databaseName, tableName, fieldName string) []string {
+ if q.gCtx.GetDatabaseMetadataFunc == nil {
+ return nil
+ }
+ _, metadata, err := q.gCtx.GetDatabaseMetadataFunc(q.ctx, q.gCtx.InstanceID, databaseName)
+ if err != nil || metadata == nil {
+ return nil
+ }
+ schema := metadata.GetSchemaMetadata("")
+ if schema == nil {
+ return nil
+ }
+ var names []string
+ for _, name := range schema.ListTableNames() {
+ if name != tableName && (!q.ignoreCaseSensitive || !strings.EqualFold(name, tableName)) {
+ continue
+ }
+ for _, column := range schema.GetTable(name).GetProto().GetColumns() {
+ if strings.EqualFold(column.Name, fieldName) {
+ names = append(names, column.Name)
+ }
+ }
+ }
+ return names
+}
+
+// isAggregateFunction reports whether MySQL resolves a HAVING name inside the
+// function as a GROUP BY name. ANY_VALUE is not one of them.
+func isAggregateFunction(name string) bool {
+ switch strings.ToLower(name) {
+ case "avg", "bit_and", "bit_or", "bit_xor", "count", "group_concat",
+ "json_arrayagg", "json_objectagg", "max", "min", "std", "stddev",
+ "stddev_pop", "stddev_samp", "sum", "var_pop", "var_samp", "variance":
+ return true
+ default:
+ return false
+ }
+}
+
+func mergeSourceColumnSetInto(dst, src base.SourceColumnSet) {
+ for column := range src {
+ dst[column] = true
+ }
+}
diff --git a/backend/plugin/parser/mysql/query_span_predicate_test.go b/backend/plugin/parser/mysql/query_span_predicate_test.go
new file mode 100644
index 0000000000..20d3bbc0ca
--- /dev/null
+++ b/backend/plugin/parser/mysql/query_span_predicate_test.go
@@ -0,0 +1,434 @@
+package mysql
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "slices"
+ "strings"
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ gomysql "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// predicateFixtureTables is the schema the span and the engines both see.
+var predicateFixtureTables = []struct {
+ name string
+ columns []string
+ ddl string
+}{
+ {name: "t", columns: []string{"a", "b", "c"}, ddl: "CREATE TABLE t (a INT, b INT, c INT)"},
+ {name: "t1", columns: []string{"a", "b", "c"}, ddl: "CREATE TABLE t1 (a INT, b INT, c INT)"},
+ {name: "u", columns: []string{"b", "c"}, ddl: "CREATE TABLE u (b INT, c INT)"},
+ {name: "j", columns: []string{"id", "doc"}, ddl: "CREATE TABLE j (id INT, doc JSON)"},
+}
+
+// predicateCase is a statement with the predicate columns its span must
+// record. TestQuerySpanPredicateColumnsMatchEngines also runs it on MySQL and
+// MariaDB, where the columns the engine reads must be exactly the result
+// lineage, the predicate columns, and other.
+type predicateCase struct {
+ name string
+ // statement may name the database as {db}.
+ statement string
+ want []string
+ // results, if set, is each result column's lineage.
+ results [][]string
+ // other lists columns the engine reads to group or order rows, which are
+ // neither result lineage nor predicates.
+ other []string
+ // unchecked lists predicate columns the engine compares without checking
+ // the privilege to read them: those a USING or NATURAL join merges.
+ unchecked []string
+ // rejected means both engines refuse the statement.
+ rejected bool
+ // skip names an engine whose answer cannot be compared, and why.
+ skip map[string]string
+}
+
+var (
+ noMariaDBLateral = map[string]string{"mariadb": "MariaDB has no LATERAL"}
+ mariaDBReadsUnusedCTE = map[string]string{"mariadb": "MariaDB checks privileges in a CTE it never runs"}
+ noMariaDBExplainAnalyze = map[string]string{"mariadb": "MariaDB spells it ANALYZE SELECT"}
+)
+
+var predicateCases = []predicateCase{
+ // WHERE.
+ {name: "where column", statement: "SELECT a FROM t WHERE b = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},
+ {name: "where function", statement: "SELECT a FROM t WHERE ascii(substr(c, 1, 1)) > 70", want: []string{"t.c"}},
+ {name: "where row comparison", statement: "SELECT a FROM t WHERE (b, c) > (1, 2)", want: []string{"t.b", "t.c"}},
+ {name: "where between in like", statement: "SELECT a FROM t WHERE b BETWEEN 1 AND c AND a IN (1, 2) AND c LIKE '1%'", want: []string{"t.a", "t.b", "t.c"}},
+ {name: "where case cast", statement: "SELECT 1 FROM t WHERE CASE WHEN b > 1 THEN CAST(c AS CHAR) ELSE '' END = '1'", want: []string{"t.b", "t.c"}},
+ {name: "where null-safe and regexp", statement: "SELECT 1 FROM t WHERE b <=> NULL OR c REGEXP '^1'", want: []string{"t.b", "t.c"}},
+ {name: "where user variable", statement: "SELECT a FROM t WHERE b = @guess", want: []string{"t.b"}},
+ {name: "where json path", statement: "SELECT id FROM j WHERE doc->>'$.k' = 'v'", want: []string{"j.doc"}, skip: map[string]string{"mariadb": "MariaDB has no ->> operator"}},
+ {name: "where member of", statement: "SELECT id FROM j WHERE 1 MEMBER OF (doc->'$.arr')", want: []string{"j.doc"}, skip: map[string]string{"mariadb": "MariaDB has no MEMBER OF"}},
+ {name: "where database-qualified column", statement: "SELECT t.a FROM t WHERE {db}.t.b = 1", want: []string{"t.b"}},
+ {name: "where column in other case", statement: "SELECT A FROM t WHERE B = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},
+ {name: "where quoted identifiers", statement: "SELECT `a` FROM `t` WHERE `b` = 1", want: []string{"t.b"}},
+ {name: "count with where", statement: "SELECT count(*) FROM t WHERE c = 0", want: []string{"t.c"}, results: [][]string{{}}},
+ {name: "star with where", statement: "SELECT * FROM t WHERE b = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}, {"t.b"}, {"t.c"}}},
+ {name: "select list only", statement: "SELECT b FROM t", want: []string{}},
+ {name: "condition in select list", statement: "SELECT CASE WHEN b = 1 THEN 1 END AS x FROM t", want: []string{}, results: [][]string{{"t.b"}}},
+ {name: "limit values", statement: "SELECT a FROM t LIMIT 1 OFFSET 2", want: []string{}},
+
+ // Joins.
+ {name: "join on", statement: "SELECT t.a FROM t JOIN t1 ON t.b = t1.b", want: []string{"t.b", "t1.b"}},
+ {name: "left join on", statement: "SELECT t.a FROM t LEFT JOIN t1 ON t1.a = t.a AND t1.c = 0", want: []string{"t.a", "t1.a", "t1.c"}},
+ {name: "join using", statement: "SELECT t.a FROM t JOIN t1 USING (b)", want: []string{"t.b", "t1.b"}, unchecked: []string{"t.b", "t1.b"}},
+ {name: "natural join", statement: "SELECT 1 FROM t NATURAL JOIN u", want: []string{"t.b", "t.c", "u.b", "u.c"}, unchecked: []string{"t.b", "t.c", "u.b", "u.c"}},
+ {name: "right join using column", statement: "SELECT b FROM t RIGHT JOIN t1 USING (b) WHERE b = 99", want: []string{"t.b", "t1.b"}, results: [][]string{{"t.b", "t1.b"}}, unchecked: []string{"t.b"}},
+ {name: "straight join", statement: "SELECT t.a FROM t STRAIGHT_JOIN t1 ON t.a = t1.a", want: []string{"t.a", "t1.a"}},
+ {name: "nested join", statement: "SELECT 1 FROM t JOIN (t1 JOIN u ON t1.b = u.b) ON t.a = t1.a", want: []string{"t.a", "t1.a", "t1.b", "u.b"}},
+ {name: "comma join", statement: "SELECT 1 FROM t, t1 WHERE t.a = t1.a", want: []string{"t.a", "t1.a"}},
+ {name: "alias named like another table", statement: "SELECT x.a FROM t AS x JOIN t1 AS t ON x.a = t.a WHERE t.b = 1", want: []string{"t.a", "t1.a", "t1.b"}, results: [][]string{{"t.a"}}},
+ {name: "correlated subquery in on", statement: "SELECT 1 FROM t JOIN t1 ON t1.a = (SELECT max(c) FROM u WHERE u.b = t.b)", want: []string{"t.b", "t1.a", "u.b", "u.c"}},
+ {name: "cross join", statement: "SELECT t.a FROM t CROSS JOIN t1", want: []string{}, results: [][]string{{"t.a"}}},
+ {name: "ambiguous column", statement: "SELECT b FROM t JOIN t1 ON t.a = t1.a", rejected: true},
+
+ // Nested queries.
+ {name: "in subquery", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM t1 WHERE b > 0)", want: []string{"t.a", "t1.a", "t1.b"}},
+ {name: "correlated exists", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE t1.a = t.a AND t1.c = 0)", want: []string{"t.a", "t1.a", "t1.c"}},
+ {name: "exists star is not read", statement: "SELECT a FROM t WHERE EXISTS (SELECT * FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}},
+ {name: "exists output decides nothing", statement: "SELECT a FROM t WHERE EXISTS (SELECT b FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}, other: []string{"t1.b"}},
+ {name: "exists wrapped in a scalar condition", statement: "SELECT 1 WHERE (SELECT EXISTS (SELECT b FROM t))", want: []string{}, other: []string{"t.b"}},
+ {name: "condition in a wrapped exists", statement: "SELECT 1 WHERE (SELECT EXISTS (SELECT 1 FROM t WHERE b = 1))", want: []string{"t.b"}},
+ {name: "exists in the select list keeps its lineage", statement: "SELECT EXISTS (SELECT b FROM t) AS e", want: []string{}, results: [][]string{{"t.b"}}},
+ {name: "scalar subquery", statement: "SELECT a FROM t WHERE b = (SELECT max(b) FROM t1)", want: []string{"t.b", "t1.b"}},
+ {name: "any subquery", statement: "SELECT a FROM t WHERE b > ANY (SELECT b FROM t1 WHERE t1.c = 0)", want: []string{"t.b", "t1.b", "t1.c"}},
+ {name: "subquery name resolves inside first", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE b = 10)", want: []string{"t1.b"}},
+ {name: "subquery name resolves outside", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM u WHERE u.b = a)", want: []string{"t.a", "u.b"}},
+ {name: "two levels out", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE EXISTS (SELECT 1 FROM u WHERE u.b = t.b AND u.c = t1.c))", want: []string{"t.b", "t1.c", "u.b", "u.c"}},
+ {name: "innermost of two levels first", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE EXISTS (SELECT 1 FROM u WHERE b = 1))", want: []string{"u.b"}},
+ {name: "condition of select list subquery", statement: "SELECT (SELECT count(*) FROM t1 WHERE t1.c = 0) AS k FROM t", want: []string{"t1.c"}, results: [][]string{{}}},
+ {name: "select list subquery resolves inside first", statement: "SELECT (SELECT b FROM t1 LIMIT 1) AS x FROM t", want: []string{}, results: [][]string{{"t1.b"}}},
+ {name: "derived table", statement: "SELECT d.a FROM (SELECT a, c FROM t WHERE b = 1) d WHERE d.c = 0", want: []string{"t.b", "t.c"}, results: [][]string{{"t.a"}}},
+ {name: "derived table column list", statement: "SELECT x FROM (SELECT a, c FROM t) d (x, y) WHERE y = 0", want: []string{"t.c"}, results: [][]string{{"t.a"}}},
+ {name: "lateral", statement: "SELECT l.k FROM t, LATERAL (SELECT count(*) AS k FROM t1 WHERE t1.a = t.a) l", want: []string{"t.a", "t1.a"}, results: [][]string{{}}, skip: noMariaDBLateral},
+ {name: "lateral in a join sees the left operand", statement: "SELECT x.b FROM t JOIN LATERAL (SELECT t.b) AS x ON TRUE", want: []string{}, results: [][]string{{"t.b"}}, skip: noMariaDBLateral},
+ {name: "lateral in a join condition", statement: "SELECT x.k FROM t JOIN LATERAL (SELECT count(*) AS k FROM t1 WHERE t1.a = t.a) AS x ON TRUE", want: []string{"t.a", "t1.a"}, results: [][]string{{}}, skip: noMariaDBLateral},
+ {name: "lateral after two joined tables", statement: "SELECT x.k FROM t JOIN t1 ON t.a = t1.a JOIN LATERAL (SELECT u.c AS k FROM u WHERE u.b = t1.b) AS x ON TRUE", want: []string{"t.a", "t1.a", "t1.b", "u.b"}, results: [][]string{{"u.c"}}, skip: noMariaDBLateral},
+ {name: "derived table cannot see its siblings", statement: "SELECT 1 FROM t, (SELECT a FROM t1 WHERE t1.b = t.b) d", rejected: true},
+ {name: "every values row", statement: "SELECT * FROM (VALUES ROW(1), ROW((SELECT b FROM t LIMIT 1))) AS v (x)", want: []string{}, results: [][]string{{"t.b"}}, skip: map[string]string{"mariadb": "MariaDB has no VALUES ROW"}},
+ {name: "window in nested condition", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM (SELECT a, ROW_NUMBER() OVER (ORDER BY b) AS r FROM t1) x WHERE r = 1)", want: []string{"t.a", "t1.a", "t1.b"}},
+
+ // CTEs.
+ {name: "cte", statement: "WITH c AS (SELECT a, b FROM t WHERE c = 0) SELECT a FROM c WHERE b = 1", want: []string{"t.b", "t.c"}, results: [][]string{{"t.a"}}},
+ {name: "unread cte", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT a FROM t1", want: []string{}, results: [][]string{{"t1.a"}}, skip: mariaDBReadsUnusedCTE},
+ {name: "cte read in subquery", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT 1 FROM t1 WHERE t1.a IN (SELECT a FROM c)", want: []string{"t.a", "t.b", "t1.a"}},
+ {name: "cte read by cte", statement: "WITH c1 AS (SELECT a FROM t WHERE b = 1), c2 AS (SELECT a FROM c1) SELECT a FROM c2", want: []string{"t.b"}, results: [][]string{{"t.a"}}},
+ {name: "cte unread by the other", statement: "WITH c1 AS (SELECT a FROM t WHERE b = 1), c2 AS (SELECT a FROM t1 WHERE c = 0) SELECT a FROM c2", want: []string{"t1.c"}, skip: mariaDBReadsUnusedCTE},
+ {name: "recursive cte", statement: "WITH RECURSIVE r (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < (SELECT max(a) FROM t)) SELECT n FROM r", want: []string{"t.a"}, results: [][]string{{}}},
+
+ // Set operations.
+ {name: "union arm condition", statement: "SELECT a FROM t WHERE b = 1 UNION SELECT a FROM t1", want: []string{"t.b"}, results: [][]string{{"t.a", "t1.a"}}},
+ {name: "union filters nothing", statement: "SELECT a FROM t UNION SELECT b FROM t1", want: []string{}, results: [][]string{{"t.a", "t1.b"}}},
+ {name: "intersect", statement: "SELECT a FROM t INTERSECT SELECT b FROM t1", want: []string{"t.a", "t1.b"}},
+ {name: "except", statement: "SELECT a FROM t EXCEPT SELECT b FROM t1", want: []string{"t.a", "t1.b"}},
+ {name: "set operation limit", statement: "SELECT a FROM t UNION ALL SELECT a FROM t1 LIMIT 1", want: []string{}},
+ {name: "parenthesized query", statement: "(SELECT a FROM t WHERE b = 1) ORDER BY a LIMIT 1", want: []string{"t.b"}},
+
+ // HAVING.
+ {name: "having alias of an aggregate", statement: "SELECT a, count(*) AS n FROM t GROUP BY a HAVING n > 1", want: []string{}, results: [][]string{{"t.a"}, {}}},
+ {name: "having alias over a from column", statement: "SELECT c AS b FROM t HAVING b > 150", want: []string{"t.c"}, results: [][]string{{"t.c"}}},
+ {name: "having group by column over an alias", statement: "SELECT c AS b FROM t GROUP BY b, c HAVING b > 150", want: []string{"t.b"}, skip: map[string]string{"mariadb": "MariaDB refuses the ambiguous name"}},
+ {name: "having aggregate argument reads from", statement: "SELECT c AS b, count(*) AS k FROM t GROUP BY c HAVING max(b) > 150", want: []string{"t.b"}},
+ {name: "having any_value reads the alias", statement: "SELECT c AS b FROM t GROUP BY c HAVING any_value(b) > 150", want: []string{"t.c"}, skip: map[string]string{"mariadb": "MariaDB has no ANY_VALUE"}},
+ {name: "having group_concat order", statement: "SELECT a FROM t GROUP BY a HAVING group_concat(b ORDER BY c) <> ''", want: []string{"t.b", "t.c"}},
+ {name: "having subquery sees from before alias", statement: "SELECT c AS b FROM u HAVING (SELECT b) > 150", want: []string{"u.b"}, results: [][]string{{"u.c"}}},
+ {name: "having alias only inside aggregate", statement: "SELECT c AS x FROM u GROUP BY c HAVING max(x) > 1", want: []string{"u.c"}},
+ {name: "having select item across join", statement: "SELECT t1.b FROM t JOIN t1 ON t.a = t1.a HAVING b > 5", want: []string{"t.a", "t1.a", "t1.b"}},
+ {name: "having plain column", statement: "SELECT b FROM t HAVING b > 1", want: []string{"t.b"}},
+ {name: "having qualified column", statement: "SELECT c AS b FROM t GROUP BY c HAVING t.c > 1", want: []string{"t.c"}},
+ {name: "having subquery condition", statement: "SELECT a FROM t GROUP BY a HAVING a > (SELECT count(*) FROM t1 WHERE t1.c = 0)", want: []string{"t.a", "t1.c"}},
+ {name: "having with group by position", statement: "SELECT c AS b FROM t GROUP BY 1 HAVING b > 150", want: []string{"t.c"}},
+ {name: "having duplicate alias", statement: "SELECT b AS x, c AS x FROM u HAVING x > 2", rejected: true},
+
+ // Select aliases elsewhere.
+ {name: "select subquery sees an earlier alias", statement: "SELECT c AS x, (SELECT x + 1) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.c"}}},
+ {name: "select subquery sees from before alias", statement: "SELECT c AS b, (SELECT b) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.b"}}},
+ {name: "select subquery forward alias", statement: "SELECT (SELECT x + 1) AS s, c AS x FROM u", rejected: true},
+ {name: "where subquery alias", statement: "SELECT c AS x FROM u WHERE (SELECT x) > 150", rejected: true},
+ {name: "where alias", statement: "SELECT c AS x FROM u WHERE x > 1", rejected: true},
+ {name: "group by subquery alias", statement: "SELECT c AS x, count(*) AS k FROM u GROUP BY (SELECT x)", want: []string{}, results: [][]string{{"u.c"}, {}}},
+ {name: "group by key", statement: "SELECT count(*) FROM t GROUP BY b", want: []string{}, other: []string{"t.b"}},
+ {name: "order by key", statement: "SELECT a FROM t ORDER BY c LIMIT 1", want: []string{}, other: []string{"t.c"}},
+ {name: "order by subquery condition", statement: "SELECT a FROM t ORDER BY (SELECT max(b) FROM t1 WHERE t1.c = 0)", want: []string{"t1.c"}, other: []string{"t1.b"}},
+ {name: "order by subquery sees from before alias", statement: "SELECT c AS b FROM u ORDER BY (SELECT b) LIMIT 1", want: []string{}, other: []string{"u.b"}},
+ {name: "window keys", statement: "SELECT a, ROW_NUMBER() OVER (PARTITION BY b ORDER BY c) AS r FROM t", want: []string{}, results: [][]string{{"t.a"}, {"t.b", "t.c"}}},
+ {name: "window clause subquery", statement: "SELECT a, ROW_NUMBER() OVER w AS r FROM t WINDOW w AS (ORDER BY (SELECT max(b) FROM t1 WHERE t1.c = 0))", want: []string{"t1.c"}, other: []string{"t1.b"}},
+
+ // JSON_TABLE.
+ {name: "json_table rows", statement: "SELECT jt.v FROM j, JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt", want: []string{"j.doc"}, results: [][]string{{"j.doc"}}},
+ {name: "json_table condition", statement: "SELECT jt.v FROM j, JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt WHERE jt.v > 3", want: []string{"j.doc"}},
+
+ // More shapes.
+ {name: "not exists and not in", statement: "SELECT a FROM t WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE t1.c = t.c) AND a NOT IN (SELECT b FROM u)", want: []string{"t.a", "t.c", "t1.c", "u.b"}},
+ {name: "row in subquery", statement: "SELECT 1 FROM t WHERE (a, b) IN (SELECT a, b FROM t1)", want: []string{"t.a", "t.b", "t1.a", "t1.b"}},
+ {name: "all subquery", statement: "SELECT a FROM t WHERE b = ALL (SELECT c FROM u)", want: []string{"t.b", "u.c"}},
+ {name: "union in subquery", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM t1 UNION SELECT b FROM u WHERE c = 0)", want: []string{"t.a", "t1.a", "u.b", "u.c"}},
+ {name: "intersect in derived table", statement: "SELECT x FROM (SELECT a AS x FROM t INTERSECT SELECT b FROM t1) d", want: []string{"t.a", "t1.b"}, results: [][]string{{"t.a", "t1.b"}}},
+ {name: "boolean operators", statement: "SELECT 1 FROM t WHERE NOT b OR (c IS TRUE XOR a)", want: []string{"t.a", "t.b", "t.c"}},
+ {name: "convert", statement: "SELECT 1 FROM t WHERE CONVERT(b, CHAR) = '1'", want: []string{"t.b"}},
+ {name: "distinct", statement: "SELECT DISTINCT a FROM t WHERE b = 1", want: []string{"t.b"}},
+ {name: "select modifiers", statement: "SELECT STRAIGHT_JOIN SQL_CALC_FOUND_ROWS t.a FROM t, t1 WHERE t.b = t1.b", want: []string{"t.b", "t1.b"}},
+ {name: "cte shadows a table", statement: "WITH t AS (SELECT a FROM t1 WHERE c = 0) SELECT a FROM t WHERE a > 1", want: []string{"t1.a", "t1.c"}, results: [][]string{{"t1.a"}}},
+ {name: "cte column list", statement: "WITH c (x, y) AS (SELECT a, b FROM t) SELECT x FROM c WHERE y = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},
+ {name: "cte read twice", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT c1.a FROM c AS c1 JOIN c AS c2 ON c1.a = c2.a", want: []string{"t.a", "t.b"}},
+ {name: "json_table joined on", statement: "SELECT jt.v FROM j JOIN JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt ON jt.v = j.id", want: []string{"j.doc", "j.id"}},
+ {name: "having alias in other case", statement: "SELECT c AS X FROM u HAVING x > 1", want: []string{"u.c"}},
+ {name: "having quoted alias", statement: "SELECT c AS `my col` FROM u HAVING `my col` > 1", want: []string{"u.c"}},
+ {name: "having alias in case", statement: "SELECT c AS b FROM u HAVING CASE WHEN b > 1 THEN 1 ELSE 0 END = 1", want: []string{"u.c"}},
+ {name: "having count distinct", statement: "SELECT a FROM t GROUP BY a HAVING count(DISTINCT b) > 1", want: []string{"t.b"}},
+ {name: "having with rollup", statement: "SELECT a, count(*) AS n FROM t GROUP BY a WITH ROLLUP HAVING a IS NULL", want: []string{"t.a"}},
+ {name: "having alias of a joined column", statement: "SELECT t1.c AS b FROM t JOIN t1 ON t.a = t1.a HAVING b > 1", want: []string{"t.a", "t1.a", "t1.c"}},
+ {name: "order by alias", statement: "SELECT c AS b FROM u ORDER BY b LIMIT 1", want: []string{}, results: [][]string{{"u.c"}}},
+ {name: "order by expression reads from first", statement: "SELECT c AS b FROM u ORDER BY b + 0 LIMIT 1", want: []string{}, other: []string{"u.b"}},
+ {name: "group by reads from first", statement: "SELECT max(b) AS c FROM u GROUP BY c", want: []string{}, results: [][]string{{"u.b"}}, other: []string{"u.c"}},
+ {name: "exists over a qualified star", statement: "SELECT a FROM t WHERE EXISTS (SELECT t1.* FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}},
+ {name: "derived table in a subquery sees the outer query", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM (SELECT b FROM t1 WHERE t1.a = t.a) d)", want: []string{"t.a", "t1.a"}, skip: map[string]string{"mariadb": "MariaDB refuses an outer reference in a derived table"}},
+ {name: "subquery two levels down sees an alias", statement: "SELECT c AS x, (SELECT (SELECT x + 1)) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.c"}}},
+
+ // EXPLAIN ANALYZE runs the query and reports the rows each step returns.
+ {name: "explain analyze condition", statement: "EXPLAIN ANALYZE SELECT 1 FROM t WHERE b = 123", want: []string{"t.b"}, skip: noMariaDBExplainAnalyze},
+ {name: "explain analyze having alias", statement: "EXPLAIN ANALYZE SELECT c AS b FROM t HAVING b > 150", want: []string{"t.c"}, skip: noMariaDBExplainAnalyze},
+ {name: "explain analyze nested condition", statement: "EXPLAIN ANALYZE SELECT a FROM t WHERE a IN (SELECT a FROM t1 WHERE c = 0)", want: []string{"t.a", "t1.a", "t1.c"}, skip: noMariaDBExplainAnalyze},
+
+ // A view's own conditions are not the caller's; its columns trace to the
+ // table. The engines check a view's privileges on the view, so they skip it.
+ {name: "view column", statement: "SELECT a FROM v WHERE vb = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}, skip: map[string]string{"mysql": "privileges are checked on the view", "mariadb": "privileges are checked on the view"}},
+}
+
+func TestQuerySpanPredicateColumns(t *testing.T) {
+ for _, tc := range predicateCases {
+ t.Run(tc.name, func(t *testing.T) {
+ span, err := predicateCaseSpan("db", strings.ReplaceAll(tc.statement, "{db}", "db"))
+ if tc.rejected {
+ return
+ }
+ require.NoError(t, err)
+ require.Nil(t, span.NotFoundError)
+ require.Equal(t, sortedOrEmpty(tc.want), columnNames(span.PredicateColumns))
+ if tc.results != nil {
+ var results [][]string
+ for _, result := range span.Results {
+ results = append(results, columnNames(result.SourceColumns))
+ }
+ want := make([][]string, 0, len(tc.results))
+ for _, columns := range tc.results {
+ want = append(want, sortedOrEmpty(columns))
+ }
+ require.Equal(t, want, results)
+ }
+ })
+ }
+}
+
+func TestQuerySpanPredicateColumnsFallBackByName(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"unknown table in a subquery", "SELECT a FROM t WHERE a IN (SELECT x FROM missing)", []string{"t.a"}},
+ {"unknown table in an exists", "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM missing WHERE missing.b = t.b)", []string{"t.b"}},
+ {"unknown column", "SELECT a FROM t WHERE b = 1 AND missing = 2", []string{"t.b"}},
+ {"unknown table in an order by subquery", "SELECT a FROM t ORDER BY (SELECT 1 FROM missing WHERE c = 0)", []string{"t.c"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ span, err := predicateCaseSpan("db", tc.statement)
+ require.NoError(t, err)
+ require.Nil(t, span.NotFoundError)
+ require.Equal(t, tc.want, columnNames(span.PredicateColumns))
+ })
+ }
+}
+
+// TestQuerySpanPredicateColumnsMatchEngines runs each case on a user denied
+// one fixture column at a time: the engine refuses exactly the statements
+// that read the column, whichever name or clause the statement reads it by.
+func TestQuerySpanPredicateColumnsMatchEngines(t *testing.T) {
+ for _, engine := range []struct {
+ name string
+ newDatabase func(testing.TB) (*testcontainer.Container, string)
+ }{
+ {name: "mysql", newDatabase: testcontainer.NewMySQLDatabase},
+ {name: "mariadb", newDatabase: testcontainer.NewMariaDBDatabase},
+ } {
+ t.Run(engine.name, func(t *testing.T) {
+ container, database := engine.newDatabase(t)
+ oracle := newColumnReadOracle(t, container, database)
+ for _, tc := range predicateCases {
+ t.Run(tc.name, func(t *testing.T) {
+ if reason, ok := tc.skip[engine.name]; ok {
+ t.Skip(reason)
+ }
+ statement := strings.ReplaceAll(tc.statement, "{db}", database)
+ read, err := oracle.columnsRead(statement)
+ if tc.rejected {
+ require.ErrorIs(t, err, errRejected)
+ return
+ }
+ require.NoError(t, err)
+
+ span, err := predicateCaseSpan(database, statement)
+ require.NoError(t, err)
+ seen := make(base.SourceColumnSet)
+ for _, result := range span.Results {
+ mergeSourceColumnSetInto(seen, result.SourceColumns)
+ }
+ mergeSourceColumnSetInto(seen, span.PredicateColumns)
+ got := append(columnNames(seen), tc.other...)
+ got = slices.DeleteFunc(got, func(column string) bool { return slices.Contains(tc.unchecked, column) })
+ slices.Sort(got)
+ require.Equal(t, read, slices.Compact(got))
+ })
+ }
+ })
+ }
+}
+
+var errRejected = errors.New("the engine refuses the statement")
+
+type columnReadOracle struct {
+ root *sql.DB
+ // deniedOne maps each fixture column to a session that may read every
+ // fixture column but that one.
+ deniedOne map[string]*sql.DB
+}
+
+func newColumnReadOracle(t *testing.T, container *testcontainer.Container, database string) *columnReadOracle {
+ t.Helper()
+ open := func(user, password string) *sql.DB {
+ db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", user, password, container.GetHost(), container.GetPort(), database))
+ require.NoError(t, err)
+ db.SetMaxOpenConns(1)
+ t.Cleanup(func() { require.NoError(t, db.Close()) })
+ return db
+ }
+ oracle := &columnReadOracle{
+ root: open(container.GetUsername(), container.GetPassword()),
+ deniedOne: make(map[string]*sql.DB),
+ }
+ for _, table := range predicateFixtureTables {
+ _, err := oracle.root.Exec(table.ddl)
+ require.NoError(t, err)
+ }
+ _, err := oracle.root.Exec("CREATE VIEW v AS SELECT a, b AS vb FROM t WHERE c > 0")
+ require.NoError(t, err)
+
+ i := 0
+ for _, denied := range predicateFixtureTables {
+ for _, deniedColumn := range denied.columns {
+ i++
+ // Users are server-wide, so they carry the database's name.
+ user := fmt.Sprintf("%s_%d", database, i)
+ _, err := oracle.root.Exec(fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY 'pw'", user))
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _, err := container.GetDB().Exec(fmt.Sprintf("DROP USER '%s'@'%%'", user))
+ require.NoError(t, err)
+ })
+ for _, table := range predicateFixtureTables {
+ grant := fmt.Sprintf("GRANT SELECT ON `%s`.`%s` TO '%s'@'%%'", database, table.name, user)
+ if table.name == denied.name {
+ var allowed []string
+ for _, column := range table.columns {
+ if column != deniedColumn {
+ allowed = append(allowed, "`"+column+"`")
+ }
+ }
+ grant = fmt.Sprintf("GRANT SELECT (%s) ON `%s`.`%s` TO '%s'@'%%'", strings.Join(allowed, ", "), database, table.name, user)
+ }
+ _, err := oracle.root.Exec(grant)
+ require.NoError(t, err)
+ }
+ oracle.deniedOne[denied.name+"."+deniedColumn] = open(user, "pw")
+ }
+ }
+ return oracle
+}
+
+// columnsRead returns the fixture columns the engine reads to run statement,
+// or errRejected if it refuses the statement outright.
+func (o *columnReadOracle) columnsRead(statement string) ([]string, error) {
+ if err := drain(o.root, statement); err != nil {
+ return nil, errors.Wrap(errRejected, err.Error())
+ }
+ read := []string{}
+ for column, db := range o.deniedOne {
+ err := drain(db, statement)
+ if err == nil {
+ continue
+ }
+ var mysqlErr *gomysql.MySQLError
+ // 1142 is a table-level refusal, which * over the table raises.
+ if errors.As(err, &mysqlErr) && (mysqlErr.Number == 1142 || mysqlErr.Number == 1143) {
+ read = append(read, column)
+ continue
+ }
+ return nil, errors.Wrapf(err, "denied %s", column)
+ }
+ slices.Sort(read)
+ return read, nil
+}
+
+func drain(db *sql.DB, statement string) error {
+ rows, err := db.Query(statement)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+ for rows.Next() {
+ }
+ return rows.Err()
+}
+
+func predicateCaseSpan(database, statement string) (*base.QuerySpan, error) {
+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{predicateFixtureMetadata(database)})
+ return GetQuerySpan(context.Background(), base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: getter,
+ ListDatabaseNamesFunc: lister,
+ Engine: storepb.Engine_MYSQL,
+ }, base.Statement{Text: statement}, database, "", false)
+}
+
+func predicateFixtureMetadata(database string) *metadatapb.DatabaseSchemaMetadata {
+ schema := &metadatapb.SchemaMetadata{
+ Views: []*metadatapb.ViewMetadata{{Name: "v", Definition: "SELECT a, b AS vb FROM t WHERE c > 0"}},
+ }
+ for _, table := range predicateFixtureTables {
+ metadata := &metadatapb.TableMetadata{Name: table.name}
+ for _, column := range table.columns {
+ metadata.Columns = append(metadata.Columns, &metadatapb.ColumnMetadata{Name: column})
+ }
+ schema.Tables = append(schema.Tables, metadata)
+ }
+ return &metadatapb.DatabaseSchemaMetadata{Name: database, Schemas: []*metadatapb.SchemaMetadata{schema}}
+}
+
+// columnNames returns the table.column names in columns, sorted, skipping
+// table-level entries.
+func columnNames(columns base.SourceColumnSet) []string {
+ names := []string{}
+ for column := range columns {
+ if column.Column != "" {
+ names = append(names, column.Table+"."+column.Column)
+ }
+ }
+ slices.Sort(names)
+ return names
+}
+
+func sortedOrEmpty(names []string) []string {
+ sorted := append([]string{}, names...)
+ slices.Sort(sorted)
+ return sorted
+}
diff --git a/backend/plugin/parser/mysql/test-data/query-span/standard.yaml b/backend/plugin/parser/mysql/test-data/query-span/standard.yaml
index 789a4d03f2..be674adb9e 100644
--- a/backend/plugin/parser/mysql/test-data/query-span/standard.yaml
+++ b/backend/plugin/parser/mysql/test-data/query-span/standard.yaml
@@ -73,7 +73,12 @@
schema: ""
table: products
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: products
+ column: product_info
- description: Test For json table
statement: |-
SELECT *
@@ -154,7 +159,12 @@
schema: ""
table: products
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: products
+ column: product_info
- description: Test For alias.* with join
statement: SELECT t1.*, t2.* FROM t t1 JOIN (select * from t) t2 ON t1.a = t2.a;
defaultDatabase: db
@@ -206,7 +216,12 @@
schema: ""
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: t
+ column: a
- description: Test For Explain statements
statement: EXPLAIN SELECT 1;
defaultDatabase: db
@@ -635,7 +650,12 @@
schema: ""
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: t
+ column: a
- description: Correlated subquery in the target field clause.
statement: SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total_paintings FROM galleries g;
defaultDatabase: db
@@ -707,7 +727,17 @@
schema: ""
table: paintings
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: galleries
+ column: id
+ - server: ""
+ database: db
+ schema: ""
+ table: paintings
+ column: gallery_id
- description: Test for functions
statement: SELECT max(a), a-b AS c1, a=b AS c2, a>b, b in (a, c, d) from (SELECT * FROM t) result;
defaultDatabase: db
@@ -1025,7 +1055,12 @@
schema: ""
table: t
column: ""
- predicatecolumns: []
+ predicatecolumns:
+ - server: ""
+ database: db
+ schema: ""
+ table: t
+ column: a
- description: Simple CTE
statement: WITH t1 AS (SELECT * FROM t) SELECT * FROM t1;
defaultDatabase: db