Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(masking)!: refuse MySQL and MariaDB queries whose conditions read masked columns (#21712)
rebelice2450df0e7693
diff --git a/backend/api/v1/predicate_columns_check.go b/backend/api/v1/predicate_columns_check.goindex e009bd276c..87c98755e0 100644--- a/backend/api/v1/predicate_columns_check.go+++ b/backend/api/v1/predicate_columns_check.go@@ -147,5 +147,10 @@ func (m *maskingLevelEvaluator) isSensitiveColumn(database *store.DatabaseMessag}func isPredicateColumnsCheckEnabled(engine storepb.Engine) bool {- return engine == storepb.Engine_MSSQL || engine == storepb.Engine_POSTGRES+ switch engine {+ case storepb.Engine_MSSQL, storepb.Engine_POSTGRES, storepb.Engine_MYSQL, storepb.Engine_MARIADB:+ return true+ default:+ return false+ }}diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.goindex b523ba2943..bfe9a40805 100644--- a/backend/common/testcontainer/shared.go+++ b/backend/common/testcontainer/shared.go@@ -25,6 +25,7 @@ var (sharedTiDB = startOnce(getTiDBContainer)sharedMongo = startOnce(getMongoDBContainer)sharedOracle = startOnce(getOracleContainer)+ sharedMySQL = startOnce(GetMySQLContainer)sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))// 10.7 is the oldest MariaDB release Bytebase supports.sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))@@ -113,6 +114,10 @@ func SharedMongoDBContainer(t testing.TB) *Container { return shared(t, sharedMo// user is a DBA.func SharedOracleContainer(t testing.TB) *Container { return shared(t, sharedOracle) }+// SharedMySQLContainer is SharedPgContainer for MySQL 8.0; tests take a+// database each with NewMySQLDatabase.+func SharedMySQLContainer(t testing.TB) *Container { return shared(t, sharedMySQL) }+// SharedMariaDBContainer is SharedPgContainer for MariaDB 11.8; tests take a// database each with NewMariaDBDatabase.func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMariaDB) }@@ -160,6 +165,13 @@ func NewTiDBDatabase(t testing.TB) (*Container, string) {return createMySQLProtocolDatabase(t, SharedTiDBContainer(t))}+// NewMySQLDatabase creates an empty database for the test on the shared MySQL+// container and returns the container with the database's name.+func NewMySQLDatabase(t testing.TB) (*Container, string) {+ t.Helper()+ return createMySQLProtocolDatabase(t, SharedMySQLContainer(t))+}+// NewMariaDBDatabase creates an empty database for the test on the shared// MariaDB container and returns the container with the database's name.func NewMariaDBDatabase(t testing.TB) (*Container, string) {diff --git a/backend/plugin/parser/mysql/main_test.go b/backend/plugin/parser/mysql/main_test.gonew file mode 100644index 0000000000..0a9c4993be--- /dev/null+++ b/backend/plugin/parser/mysql/main_test.go@@ -0,0 +1,9 @@+package mysql++import (+ "testing"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+)++func TestMain(m *testing.M) { testcontainer.Main(m) }diff --git a/backend/plugin/parser/mysql/query_span_extractor.go b/backend/plugin/parser/mysql/query_span_extractor.goindex ea7902703b..ca4a6a1326 100644--- a/backend/plugin/parser/mysql/query_span_extractor.go+++ b/backend/plugin/parser/mysql/query_span_extractor.go@@ -2,6 +2,7 @@ package mysqlimport ("context"+ "slices""strconv""strings"@@ -25,8 +26,8 @@ type querySpanExtractor struct {// ctes records common table expressions visible to the current select scope.ctes []*base.PseudoTable- // outerTableSources resolves correlated subquery column references.- outerTableSources []base.TableSource+ // outerScopes resolves correlated subquery column references, innermost last.+ outerScopes []outerScope// tableSourceFrom is the set of table sources visible from the current FROM clause.tableSourceFrom []base.TableSource@@ -34,6 +35,26 @@ type querySpanExtractor struct {// priorTableInFrom resolves JSON_TABLE document expressions against preceding FROM items.priorTableInFrom []base.TableSource+ // clause, selectItems, groupBy, and inAggregate are the position in the+ // current SELECT that decides how an unqualified name resolves.+ clause clause+ selectItems []base.QuerySpanResult+ groupBy []ast.ExprNode+ inAggregate bool+ // inCondition and outputInCondition hide an EXISTS subquery's output,+ // which decides nothing: inCondition within a condition of this SELECT,+ // outputInCondition when this query's own output feeds a condition.+ inCondition bool+ outputInCondition bool++ // predicateColumns is shared with the extractors of nested queries.+ predicateColumns base.SourceColumnSet+ // ctePredicateColumns holds each CTE's predicate columns until a query+ // reads the CTE; MySQL never evaluates an unreferenced CTE.+ ctePredicateColumns map[*base.PseudoTable]base.SourceColumnSet+ // accessTables are the tables the statement reads, for the name fallback.+ accessTables base.SourceColumnSet+viewResolutionStack map[string]bool}@@ -42,6 +63,8 @@ func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContextdefaultDatabase: defaultDatabase,gCtx: gCtx,ignoreCaseSensitive: ignoreCaseSensitive,+ predicateColumns: make(base.SourceColumnSet),+ ctePredicateColumns: make(map[*base.PseudoTable]base.SourceColumnSet),}}@@ -63,127 +86,95 @@ const (NaturalRightOuterJoin)+// joinTableSources returns the columns a join exposes. A column merged by+// USING or NATURAL is COALESCE(left, right), and in matched rows both sides+// hold its value, so it carries both sides' sources.func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.TableSource {switch tp {- case Join, InnerJoin, CrossJoin, StraightJoin, LeftOuterJoin, RightOuterJoin:- var columns []base.QuerySpanResult- rightFieldsMap := make(map[string]bool)- for _, field := range r.GetQuerySpanResult() {- rightFieldsMap[strings.ToLower(field.Name)] = true- }- lowercaseUsingFields := make(map[string]bool)- for _, field := range using {- lowercaseUsingFields[strings.ToLower(field)] = true- }- for _, field := range l.GetQuerySpanResult() {- columns = append(columns, field)- if _, ok := lowercaseUsingFields[strings.ToLower(field.Name)]; ok {- delete(rightFieldsMap, strings.ToLower(field.Name))- }- }- for _, field := range r.GetQuerySpanResult() {- if _, ok := rightFieldsMap[strings.ToLower(field.Name)]; ok {- columns = append(columns, field)+ case Join, InnerJoin, CrossJoin, StraightJoin, LeftOuterJoin, RightOuterJoin,+ NaturalInnerJoin, NaturalLeftOuterJoin, NaturalRightOuterJoin:+ default:+ return nil+ }+ joined := make(map[string]bool)+ for _, name := range joinedColumnNames(l, r, tp, using) {+ joined[strings.ToLower(name)] = true+ }+ rightColumns := r.GetQuerySpanResult()+ var columns []base.QuerySpanResult+ for _, field := range l.GetQuerySpanResult() {+ if joined[strings.ToLower(field.Name)] {+ if rightField, ok := querySpanResultNamed(rightColumns, field.Name); ok {+ field.SourceColumns, _ = base.MergeSourceColumnSet(field.SourceColumns, rightField.SourceColumns)}}- return &base.PseudoTable{- Columns: columns,+ columns = append(columns, field)+ }+ for _, field := range rightColumns {+ if !joined[strings.ToLower(field.Name)] {+ columns = append(columns, field)}+ }+ return &base.PseudoTable{+ Columns: columns,+ }+}++// joinedColumnNames returns the columns a join merges: the USING list, or for+// a NATURAL join every column name both sides share.+func joinedColumnNames(l, r base.TableSource, tp joinType, using []string) []string {+ switch tp {case NaturalInnerJoin, NaturalLeftOuterJoin, NaturalRightOuterJoin:- rightFieldsMap := make(map[string]bool)- for _, field := range r.GetQuerySpanResult() {- rightFieldsMap[strings.ToLower(field.Name)] = true- }- var columns []base.QuerySpanResult+ var names []string+ rightColumns := r.GetQuerySpanResult()for _, field := range l.GetQuerySpanResult() {- columns = append(columns, field)- delete(rightFieldsMap, strings.ToLower(field.Name))- }- for _, field := range r.GetQuerySpanResult() {- if _, ok := rightFieldsMap[strings.ToLower(field.Name)]; ok {- columns = append(columns, field)+ if _, ok := querySpanResultNamed(rightColumns, field.Name); ok {+ names = append(names, field.Name)}}- return &base.PseudoTable{- Columns: columns,- }+ return namesdefault:- return nil+ return using}}-func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {- findInTableSource := func(tableSource base.TableSource) ([]base.QuerySpanResult, bool) {- if q.ignoreCaseSensitive {- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {- return nil, false- }- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {- return nil, false- }- } else {- if databaseName != "" && databaseName != tableSource.GetDatabaseName() {- return nil, false- }- if tableName != "" && tableName != tableSource.GetTableName() {- return nil, false- }+func querySpanResultNamed(results []base.QuerySpanResult, name string) (base.QuerySpanResult, bool) {+ for _, result := range results {+ if strings.EqualFold(result.Name, name) {+ return result, true}- return tableSource.GetQuerySpanResult(), true}+ return base.QuerySpanResult{}, false+}- for i := len(q.outerTableSources) - 1; i >= 0; i-- {- if querySpanResult, ok := findInTableSource(q.outerTableSources[i]); ok {- return querySpanResult, true- }- }- for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {- if querySpanResult, ok := findInTableSource(q.tableSourceFrom[i]); ok {- return querySpanResult, true+func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {+ for _, tables := range q.tableScopes() {+ for i := len(tables) - 1; i >= 0; i-- {+ if q.tableSourceMatches(tables[i], databaseName, tableName) {+ return tables[i].GetQuerySpanResult(), true+ }}}return nil, false}+// getFieldColumnSource resolves a column reference as MySQL does: the current+// SELECT first, by the rules of the clause the reference is in, then each+// enclosing SELECT outward, its tables before the select list it exposes.func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, error) {databaseName = q.filterClusterName(databaseName)- findInTableSource := func(tableSource base.TableSource) (base.SourceColumnSet, bool) {- if q.ignoreCaseSensitive {- if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {- return nil, false- }- if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {- return nil, false- }- } else {- if databaseName != "" && databaseName != tableSource.GetDatabaseName() {- return nil, false- }- if tableName != "" && tableName != tableSource.GetTableName() {- return nil, false- }- }- for _, column := range tableSource.GetQuerySpanResult() {- if strings.EqualFold(column.Name, fieldName) {- return column.SourceColumns, true- }- }- return nil, false+ if sourceColumnSet, ok := q.resolveInCurrentSelect(databaseName, tableName, fieldName); ok {+ return sourceColumnSet, nil}-- for i := len(q.outerTableSources) - 1; i >= 0; i-- {- if sourceColumnSet, ok := findInTableSource(q.outerTableSources[i]); ok {- return sourceColumnSet, nil- }- }- for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {- if sourceColumnSet, ok := findInTableSource(q.tableSourceFrom[i]); ok {+ for i := len(q.outerScopes) - 1; i >= 0; i-- {+ scope := q.outerScopes[i]+ if sourceColumnSet, ok := q.findColumnInTables(scope.tables, databaseName, tableName, fieldName); ok {return sourceColumnSet, nil}- }- for i := len(q.priorTableInFrom) - 1; i >= 0; i-- {- if sourceColumnSet, ok := findInTableSource(q.priorTableInFrom[i]); ok {- return sourceColumnSet, nil+ if databaseName == "" && tableName == "" {+ if sourceColumnSet, ok := selectItemsNamed(scope.items, fieldName); ok {+ return sourceColumnSet, nil+ }}}@@ -194,6 +185,50 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, field}}+// tableScopes returns the table sources a name can resolve to, nearest first.+func (q *querySpanExtractor) tableScopes() [][]base.TableSource {+ scopes := [][]base.TableSource{q.tableSourceFrom, q.priorTableInFrom}+ for i := len(q.outerScopes) - 1; i >= 0; i-- {+ scopes = append(scopes, q.outerScopes[i].tables)+ }+ return scopes+}++func (q *querySpanExtractor) tableSourceMatches(tableSource base.TableSource, databaseName, tableName string) bool {+ if q.ignoreCaseSensitive {+ if databaseName != "" && !strings.EqualFold(databaseName, tableSource.GetDatabaseName()) {+ return false+ }+ if tableName != "" && !strings.EqualFold(tableName, tableSource.GetTableName()) {+ return false+ }+ return true+ }+ if databaseName != "" && databaseName != tableSource.GetDatabaseName() {+ return false+ }+ if tableName != "" && tableName != tableSource.GetTableName() {+ return false+ }+ return true+}++// findColumnInTables searches tables last to first, so a join's merged+// columns, appended after its operands, win for an unqualified name.+func (q *querySpanExtractor) findColumnInTables(tables []base.TableSource, databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {+ for i := len(tables) - 1; i >= 0; i-- {+ if !q.tableSourceMatches(tables[i], databaseName, tableName) {+ continue+ }+ for _, column := range tables[i].GetQuerySpanResult() {+ if strings.EqualFold(column.Name, fieldName) {+ return column.SourceColumns, true+ }+ }+ }+ return nil, false+}+func (q *querySpanExtractor) filterClusterName(databaseName string) string {if q.gCtx.Engine == storepb.Engine_STARROCKS {list := strings.Split(databaseName, ":")@@ -209,6 +244,7 @@ func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (bafor i := len(q.ctes) - 1; i >= 0; i-- {table := q.ctes[i]if table.Name == tableName {+ mergeSourceColumnSetInto(q.predicateColumns, q.ctePredicateColumns[table])return table, nil}}@@ -434,7 +470,9 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statementexplainAnalyze := isExplain && explain.Analyzevar results []base.QuerySpanResult+ predicateColumns := base.SourceColumnSet{}if queryType == base.Select {+ q.accessTables = accessTablestableSource, err := q.extractOmniSelectRoot(root)if err != nil {var resourceNotFound *base.ResourceNotFoundError@@ -443,20 +481,21 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statementType: base.Select,SourceColumns: accessTables,Results: []base.QuerySpanResult{},- PredicateColumns: base.SourceColumnSet{},+ PredicateColumns: q.predicateColumns,NotFoundError: resourceNotFound,}, nil}return nil, err}results = tableSource.GetQuerySpanResult()+ predicateColumns = q.predicateColumns}return &base.QuerySpan{Type: queryType,ExplainAnalyze: explainAnalyze,SourceColumns: accessTables,Results: results,- PredicateColumns: base.SourceColumnSet{},+ PredicateColumns: predicateColumns,}, nil}@@ -465,6 +504,9 @@ func (q *omniQuerySpanExtractor) extractOmniSelectRoot(root ast.Node) (*base.Psecase *ast.SelectStmt:return q.extractOmniSelectStmt(n)case *ast.ExplainStmt:+ if n.Analyze {+ q.collectExplainedQueryColumns(n.Stmt)+ }return &base.PseudoTable{}, nilcase *ast.TableStmt:return q.extractOmniTableStmt(n)@@ -486,38 +528,74 @@ func (q *omniQuerySpanExtractor) extractOmniSelectStmt(stmt *ast.SelectStmt) (*boriginalTableSourceFrom := q.tableSourceFromoriginalPriorLength := len(q.priorTableInFrom)originalCTELength := len(q.ctes)+ originalPosition := q.position()defer func() {q.tableSourceFrom = originalTableSourceFromq.priorTableInFrom = q.priorTableInFrom[:originalPriorLength]q.ctes = q.ctes[:originalCTELength]+ q.setPosition(originalPosition)}()+ q.setPosition(selectPosition{})if err := q.processOmniCTEs(stmt.CTEs); err != nil {return nil, err}- if inner := stmt.ParenSource; inner != nil {- return q.extractOmniSelectStmt(inner)- }- if tableStmt := stmt.TableSource; tableStmt != nil {- return q.extractOmniTableStmt(tableStmt)- }- if valuesStmt := stmt.ValuesSource; valuesStmt != nil {- return q.extractOmniValuesStmt(valuesStmt)+ var table *base.PseudoTable+ var err error+ switch {+ case stmt.ParenSource != nil:+ table, err = q.extractOmniSelectStmt(stmt.ParenSource)+ case stmt.TableSource != nil:+ table, err = q.extractOmniTableStmt(stmt.TableSource)+ case stmt.ValuesSource != nil:+ table, err = q.extractOmniValuesStmt(stmt.ValuesSource)+ case stmt.SetOp != ast.SetOpNone:+ table, err = q.extractOmniSetOp(stmt)+ default:+ return q.extractOmniQuerySpecification(stmt)}- if stmt.SetOp != ast.SetOpNone {- return q.extractOmniSetOp(stmt)+ if err != nil {+ return nil, err}+ // The ORDER BY and LIMIT written after a parenthesized query, a TABLE or+ // VALUES primary, or a set operation belong to the wrapper.+ q.selectItems = table.GetQuerySpanResult()+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)+ return table, nil+}+// extractOmniQuerySpecification extracts a SELECT ... FROM ... block,+// recording the columns its conditions read as it resolves each clause.+func (q *omniQuerySpanExtractor) extractOmniQuerySpecification(stmt *ast.SelectStmt) (*base.PseudoTable, error) {+ q.clause = clauseFromfromSources, err := q.extractOmniTableSources(stmt.From)if err != nil {return nil, err}q.tableSourceFrom = append(q.tableSourceFrom, fromSources...)+ q.clause = clauseWhere+ q.collectConditionColumns(stmt.Where)++ q.clause = clauseSelectListresults, err := q.extractOmniTargetList(stmt.TargetList, fromSources)if err != nil {return nil, err}+ q.selectItems = results++ q.groupBy = stmt.GroupBy+ q.clause = clauseGroupBy+ for _, expr := range stmt.GroupBy {+ q.collectNestedQueryColumns(expr)+ }+ q.clause = clauseHaving+ q.collectConditionColumns(stmt.Having)+ q.clause = clauseOrderBy+ for _, window := range stmt.WindowClause {+ q.collectNestedQueryColumns(window)+ }+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)return &base.PseudoTable{Columns: results,}, nil@@ -534,21 +612,36 @@ func (q *omniQuerySpanExtractor) extractOmniTableStmt(stmt *ast.TableStmt) (*basif err != nil {return nil, err}+ q.selectItems = tableSource.GetQuerySpanResult()+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)return &base.PseudoTable{Columns: tableSource.GetQuerySpanResult()}, nil}+// extractOmniValuesStmt merges every row into its column, since each row is+// returned.func (q *omniQuerySpanExtractor) extractOmniValuesStmt(stmt *ast.ValuesStmt) (*base.PseudoTable, error) {if stmt == nil || len(stmt.Rows) == 0 {return &base.PseudoTable{}, nil}var columns []base.QuerySpanResult- for _, expr := range stmt.Rows[0] {- field, err := q.extractOmniExpr(expr)- if err != nil {- return nil, err+ for i, row := range stmt.Rows {+ for j, expr := range row {+ field, err := q.extractOmniExpr(expr)+ if err != nil {+ return nil, err+ }+ if i == 0 {+ columns = append(columns, field)+ continue+ }+ if j < len(columns) {+ columns[j].SourceColumns, _ = base.MergeSourceColumnSet(columns[j].SourceColumns, field.SourceColumns)+ columns[j].IsPlainField = columns[j].IsPlainField && field.IsPlainField+ }}- columns = append(columns, field)}+ q.selectItems = columns+ q.collectTrailingClauses(stmt.OrderBy, stmt.Limit)return &base.PseudoTable{Columns: columns}, nil}@@ -569,9 +662,15 @@ func (q *omniQuerySpanExtractor) extractOmniSetOp(stmt *ast.SelectStmt) (*base.Pif len(leftResults) != len(rightResults) {return nil, errors.Errorf("MySQL UNION operator left has %d fields, right has %d fields", len(leftResults), len(rightResults))}+ // INTERSECT and EXCEPT keep a row by whether the other branch has it, so+ // both branches' output decides which rows come back.+ filters := stmt.SetOp == ast.SetOpIntersect || stmt.SetOp == ast.SetOpExceptresults := make([]base.QuerySpanResult, 0, len(leftResults))for i := range leftResults {sourceColumns, _ := base.MergeSourceColumnSet(leftResults[i].SourceColumns, rightResults[i].SourceColumns)+ if filters {+ mergeSourceColumnSetInto(q.predicateColumns, sourceColumns)+ }results = append(results, base.QuerySpanResult{Name: leftResults[i].Name,SourceColumns: sourceColumns,@@ -581,13 +680,20 @@ func (q *omniQuerySpanExtractor) extractOmniSetOp(stmt *ast.SelectStmt) (*base.Preturn &base.PseudoTable{Columns: results}, nil}+// processOmniCTEs extracts each CTE into its own predicate set, which+// findTableSchema merges into the reading query's.func (q *omniQuerySpanExtractor) processOmniCTEs(ctes []*ast.CommonTableExpr) error {for _, cte := range ctes {+ readerPredicateColumns := q.predicateColumns+ q.predicateColumns = make(base.SourceColumnSet)tableSource, err := q.extractOmniCTE(cte)+ ctePredicateColumns := q.predicateColumns+ q.predicateColumns = readerPredicateColumnsif err != nil {return err}if tableSource != nil {+ q.ctePredicateColumns[tableSource] = ctePredicateColumnsq.ctes = append(q.ctes, tableSource)}}@@ -743,6 +849,8 @@ func (q *omniQuerySpanExtractor) extractOmniJSONTable(expr *ast.JsonTableExpr) (if err != nil {return nil, err}+ // JSON_TABLE returns a row per match in the document.+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)var columns []base.QuerySpanResultfor _, column := range flattenOmniJSONTableColumns(expr.Columns) {columns = append(columns, base.QuerySpanResult{@@ -789,10 +897,26 @@ func (q *omniQuerySpanExtractor) extractOmniJoin(join *ast.JoinClause) (base.Tab}var using []string- if condition, ok := join.Condition.(*ast.UsingCondition); ok && condition != nil {- using = condition.Columns+ switch condition := join.Condition.(type) {+ case *ast.OnCondition:+ if condition != nil {+ q.collectConditionColumns(condition.Expr)+ }+ case *ast.UsingCondition:+ if condition != nil {+ using = condition.Columns+ }+ default:+ }+ tp := omniJoinType(join.Type)+ if left != nil && right != nil {+ // USING and NATURAL compare the joined columns of both sides.+ for _, name := range joinedColumnNames(left, right, tp, using) {+ q.recordJoinedColumn(left, name)+ q.recordJoinedColumn(right, name)+ }}- return joinTableSources(left, right, omniJoinType(join.Type), using), nil+ return joinTableSources(left, right, tp, using), nil}func omniJoinType(joinType ast.JoinType) joinType {@@ -822,10 +946,14 @@ func (q *omniQuerySpanExtractor) extractOmniSubquery(expr *ast.SubqueryExpr) (*bif expr == nil {return &base.PseudoTable{}, nil}- subqueryExtractor := q.cloneOmniForSubquery()+ // A derived table cannot see the FROM clause it sits in unless it is+ // LATERAL, and then only the items before it: earlier FROM items, and the+ // operands extractOmniJoin has already added for the join it is in.+ scope := outerScope{}if expr.Lateral {- subqueryExtractor.outerTableSources = append(subqueryExtractor.outerTableSources, q.priorTableInFrom...)+ scope.tables = slices.Concat(q.tableSourceFrom, q.priorTableInFrom)}+ subqueryExtractor := q.cloneOmniWithScope(scope)tableSource, err := subqueryExtractor.extractOmniSelectStmt(expr.Select)if err != nil {return nil, err@@ -848,16 +976,37 @@ func (q *omniQuerySpanExtractor) extractOmniSubquery(expr *ast.SubqueryExpr) (*b}, nil}+// cloneOmniForSubquery returns the extractor for a query nested in an+// expression of the current SELECT, which sees its tables and, where MySQL+// allows, its select list.func (q *omniQuerySpanExtractor) cloneOmniForSubquery() *omniQuerySpanExtractor {+ return q.cloneOmniWithScope(outerScope{+ tables: slices.Concat(q.tableSourceFrom, q.priorTableInFrom),+ items: q.selectItemsVisibleToSubquery(),+ })+}++// cloneOmniWithScope returns an extractor for a nested query that sees scope+// as its nearest enclosing SELECT. The nested query records its predicate+// columns into the same set.+func (q *omniQuerySpanExtractor) cloneOmniWithScope(scope outerScope) *omniQuerySpanExtractor {+ outerScopes := slices.Clone(q.outerScopes)+ if len(scope.tables) > 0 || len(scope.items) > 0 {+ outerScopes = append(outerScopes, scope)+ }return &omniQuerySpanExtractor{querySpanExtractor: &querySpanExtractor{ctx: q.ctx,defaultDatabase: q.defaultDatabase,gCtx: q.gCtx,ctes: q.ctes,- outerTableSources: append(q.outerTableSources, q.tableSourceFrom...),+ outerScopes: outerScopes,tableSourceFrom: []base.TableSource{},ignoreCaseSensitive: q.ignoreCaseSensitive,+ outputInCondition: q.inCondition || q.outputInCondition,+ predicateColumns: q.predicateColumns,+ ctePredicateColumns: q.ctePredicateColumns,+ accessTables: q.accessTables,viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),},source: q.source,@@ -904,6 +1053,8 @@ func (q *omniQuerySpanExtractor) extractOmniTargetList(targets []ast.ExprNode, fdefault:}+ // A subquery in this item can name the items before it.+ q.selectItems = resultfield, err := q.extractOmniTarget(target)if err != nil {return nil, err@@ -999,7 +1150,13 @@ func (q *omniQuerySpanExtractor) extractOmniExpr(expr ast.ExprNode) (base.QueryS}}}+ // HAVING resolves the arguments of an aggregate like a GROUP BY name.+ inAggregate := q.inAggregate+ if q.clause == clauseHaving && e.Over == nil && isAggregateFunction(e.Name) {+ q.inAggregate = true+ }sourceColumns, err := q.mergeOmniExprSources(exprs...)+ q.inAggregate = inAggregateif err != nil {return base.QuerySpanResult{}, err}@@ -1015,6 +1172,10 @@ func (q *omniQuerySpanExtractor) extractOmniExpr(expr ast.ExprNode) (base.QuerySif err != nil {return base.QuerySpanResult{}, err}+ if q.inCondition || q.outputInCondition {+ // EXISTS reports whether a row comes back, not what the row holds.+ sourceColumns = base.SourceColumnSet{}+ }return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nilcase *ast.CaseExpr:var exprs []ast.ExprNodediff --git a/backend/plugin/parser/mysql/query_span_extractor_test.go b/backend/plugin/parser/mysql/query_span_extractor_test.goindex 42267a4209..01f25204cd 100644--- a/backend/plugin/parser/mysql/query_span_extractor_test.go+++ b/backend/plugin/parser/mysql/query_span_extractor_test.go@@ -94,8 +94,8 @@ func TestOmniQuerySpanScaffold_QueryTypesAndAccessTables(t *testing.T) {require.Equal(t, sourceColumnSetFromResources(tc.wantResources), span.SourceColumns)if tc.wantType != base.Select {require.Empty(t, span.Results)+ require.Empty(t, span.PredicateColumns)}- require.Empty(t, span.PredicateColumns)})}}@@ -818,12 +818,17 @@ func TestOmniQuerySpanScenarioBatch2_FromJoinAndScopeCoverage(t *testing.T) {"SELECT * FROM t1 NATURAL LEFT JOIN t2","SELECT * FROM t1 NATURAL RIGHT JOIN t2",}+ // A NATURAL join merges each shared column, which holds both sides' value.want := []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}, {Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}, {Database: "db", Table: "t2", Column: "c"}}), IsPlainField: true},+ }+ wantNonNatural := []base.QuerySpanResult{{Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},{Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}}), IsPlainField: true},{Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}}), IsPlainField: true},}- wantNonNatural := append([]base.QuerySpanResult{}, want...)wantNonNatural = append(wantNonNatural,base.QuerySpanResult{Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},base.QuerySpanResult{Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},@@ -1041,17 +1046,17 @@ func TestOmniQuerySpanScenarioBatch2_SubqueryCTEAndSetCoverage(t *testing.T) {},},{- name: "unqualified_inner_column_uses_legacy_resolution_order",+ name: "unqualified_inner_column_resolves_in_the_subquery_first",statement: "SELECT (SELECT a FROM t2) AS inner_a FROM t1",want: []base.QuerySpanResult{- {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},},},{- name: "inner_alias_shadow_keeps_legacy_outer_first_lookup",+ name: "inner_alias_shadows_the_outer_alias",statement: "SELECT (SELECT x.a FROM t2 AS x) AS inner_a FROM t1 AS x",want: []base.QuerySpanResult{- {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},},},{diff --git a/backend/plugin/parser/mysql/query_span_predicate.go b/backend/plugin/parser/mysql/query_span_predicate.gonew file mode 100644index 0000000000..f4fd181d3d--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_predicate.go@@ -0,0 +1,296 @@+package mysql++import (+ "strings"++ "github.com/bytebase/omni/mysql/ast"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// clause is where a name sits in a SELECT, which decides what it can resolve+// to. query_span_predicate_test.go checks each rule against MySQL and MariaDB.+type clause int++const (+ // clauseSelectList, clauseFrom, and clauseWhere see the FROM clause only.+ clauseSelectList clause = iota+ clauseFrom+ clauseWhere+ // clauseGroupBy sees the FROM clause, then the select list.+ clauseGroupBy+ // clauseHaving sees a GROUP BY column, then the select list, then the FROM+ // clause; inside an aggregate it resolves like clauseGroupBy.+ clauseHaving+ // clauseOrderBy resolves no names itself, since the ORDER BY keys are not+ // predicates; queries nested in it see the select list.+ clauseOrderBy+)++// outerScope is what an enclosing SELECT shows a query nested in it.+type outerScope struct {+ tables []base.TableSource+ // items is the select list the nested query can name, set only where+ // MySQL lets it: the select list (items before it), GROUP BY, HAVING, and+ // ORDER BY.+ items []base.QuerySpanResult+}++// selectPosition is the per-SELECT state a nested SELECT saves and restores.+type selectPosition struct {+ clause clause+ selectItems []base.QuerySpanResult+ groupBy []ast.ExprNode+ inAggregate bool+ inCondition bool+}++func (q *querySpanExtractor) position() selectPosition {+ return selectPosition{+ clause: q.clause,+ selectItems: q.selectItems,+ groupBy: q.groupBy,+ inAggregate: q.inAggregate,+ inCondition: q.inCondition,+ }+}++func (q *querySpanExtractor) setPosition(p selectPosition) {+ q.clause = p.clause+ q.selectItems = p.selectItems+ q.groupBy = p.groupBy+ q.inAggregate = p.inAggregate+ q.inCondition = p.inCondition+}++func (q *querySpanExtractor) selectItemsVisibleToSubquery() []base.QuerySpanResult {+ switch q.clause {+ case clauseSelectList, clauseGroupBy, clauseHaving, clauseOrderBy:+ return q.selectItems+ default:+ return nil+ }+}++// resolveInCurrentSelect resolves a name in the SELECT being extracted.+func (q *querySpanExtractor) resolveInCurrentSelect(databaseName, tableName, fieldName string) (base.SourceColumnSet, bool) {+ fromClause := func(databaseName, tableName string) (base.SourceColumnSet, bool) {+ if columns, ok := q.findColumnInTables(q.tableSourceFrom, databaseName, tableName, fieldName); ok {+ return columns, true+ }+ return q.findColumnInTables(q.priorTableInFrom, databaseName, tableName, fieldName)+ }+ if databaseName != "" || tableName != "" {+ return fromClause(databaseName, tableName)+ }+ switch {+ case q.clause == clauseHaving && !q.inAggregate:+ // MySQL warns when a GROUP BY column and a select alias share the name+ // and takes the column; MariaDB refuses the query.+ if group := q.groupByColumnNamed(fieldName); group != nil {+ if columns, ok := fromClause(group.Schema, group.Table); ok {+ return columns, true+ }+ }+ if columns, ok := selectItemsNamed(q.selectItems, fieldName); ok {+ return columns, true+ }+ return fromClause("", "")+ case q.clause == clauseHaving, q.clause == clauseGroupBy:+ if columns, ok := fromClause("", ""); ok {+ return columns, true+ }+ return selectItemsNamed(q.selectItems, fieldName)+ default:+ return fromClause("", "")+ }+}++func (q *querySpanExtractor) groupByColumnNamed(name string) *ast.ColumnRef {+ for _, expr := range q.groupBy {+ if ref, ok := expr.(*ast.ColumnRef); ok && !ref.Star && strings.EqualFold(ref.Column, name) {+ return ref+ }+ }+ return nil+}++// selectItemsNamed returns the sources of every select item called name. Two+// different items with one name make MySQL refuse the query.+func selectItemsNamed(items []base.QuerySpanResult, name string) (base.SourceColumnSet, bool) {+ columns := make(base.SourceColumnSet)+ found := false+ for _, item := range items {+ if strings.EqualFold(item.Name, name) {+ mergeSourceColumnSetInto(columns, item.SourceColumns)+ found = true+ }+ }+ return columns, found+}++// collectConditionColumns records the columns expr reads, where the database+// evaluates it on unmasked values to decide which rows come back.+func (q *omniQuerySpanExtractor) collectConditionColumns(expr ast.ExprNode) {+ if expr == nil {+ return+ }+ inCondition := q.inCondition+ q.inCondition = true+ field, err := q.extractOmniExpr(expr)+ q.inCondition = inCondition+ if err != nil {+ q.recordColumnsNamedIn(expr)+ return+ }+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)+}++// collectNestedQueryColumns records the predicate columns of the queries+// nested in node, whose own value only groups or orders the rows.+func (q *omniQuerySpanExtractor) collectNestedQueryColumns(node ast.Node) {+ if node == nil {+ return+ }+ ast.Inspect(node, func(n ast.Node) bool {+ sel, ok := n.(*ast.SelectStmt)+ if !ok {+ return true+ }+ if _, _, err := q.extractOmniSelectSourceColumns(sel); err != nil {+ q.recordColumnsNamedIn(sel)+ }+ return false+ })+}++// collectTrailingClauses records what ORDER BY and LIMIT decide: the queries+// nested in ORDER BY, and the row count. The ORDER BY keys only order rows.+func (q *omniQuerySpanExtractor) collectTrailingClauses(orderBy []*ast.OrderByItem, limit *ast.Limit) {+ q.clause = clauseOrderBy+ for _, item := range orderBy {+ if item != nil {+ q.collectNestedQueryColumns(item.Expr)+ }+ }+ if limit != nil {+ q.collectConditionColumns(limit.Count)+ q.collectConditionColumns(limit.Offset)+ }+}++// collectExplainedQueryColumns records the predicate columns of the query+// EXPLAIN ANALYZE runs, whose plan reports the rows each condition kept. The+// plan is not a result set, so the query's lineage is dropped.+func (q *omniQuerySpanExtractor) collectExplainedQueryColumns(stmt ast.StmtNode) {+ if _, err := q.extractOmniSelectRoot(stmt); err != nil {+ q.recordColumnsNamedIn(stmt)+ }+}++func (q *omniQuerySpanExtractor) recordJoinedColumn(table base.TableSource, name string) {+ if field, ok := querySpanResultNamed(table.GetQuerySpanResult(), name); ok {+ mergeSourceColumnSetInto(q.predicateColumns, field.SourceColumns)+ return+ }+ q.recordColumnsNamed("", "", name)+}++// recordColumnsNamedIn over-approximates the columns node reads by name, for+// a condition the extractor could not resolve.+func (q *omniQuerySpanExtractor) recordColumnsNamedIn(node ast.Node) {+ ast.Inspect(node, func(n ast.Node) bool {+ if ref, ok := n.(*ast.ColumnRef); ok && !ref.Star {+ q.recordColumnsNamed(ref.Schema, ref.Table, ref.Column)+ }+ return true+ })+}++// recordColumnsNamed records every column called fieldName that the reference+// could mean: in any visible table or select list, and in any table the+// statement reads. A qualifier that names none of them is ignored.+func (q *omniQuerySpanExtractor) recordColumnsNamed(databaseName, tableName, fieldName string) {+ qualifierMatched := false+ recordTables := func(tables []base.TableSource) {+ for _, table := range tables {+ if !q.tableSourceMatches(table, databaseName, tableName) {+ continue+ }+ qualifierMatched = true+ for _, column := range table.GetQuerySpanResult() {+ if strings.EqualFold(column.Name, fieldName) {+ mergeSourceColumnSetInto(q.predicateColumns, column.SourceColumns)+ }+ }+ }+ }+ for _, tables := range q.tableScopes() {+ recordTables(tables)+ }+ if databaseName == "" && tableName == "" {+ columns, _ := selectItemsNamed(q.selectItems, fieldName)+ mergeSourceColumnSetInto(q.predicateColumns, columns)+ for _, scope := range q.outerScopes {+ columns, _ := selectItemsNamed(scope.items, fieldName)+ mergeSourceColumnSetInto(q.predicateColumns, columns)+ }+ }++ for access := range q.accessTables {+ if access.Table == "" {+ continue+ }+ if tableName != "" && qualifierMatched && !q.tableSourceMatches(&base.PhysicalTable{Database: access.Database, Name: access.Table}, databaseName, tableName) {+ continue+ }+ for _, column := range q.metadataColumnsNamed(access.Database, access.Table, fieldName) {+ q.predicateColumns[base.ColumnResource{Database: access.Database, Table: access.Table, Column: column}] = true+ }+ }+}++func (q *omniQuerySpanExtractor) metadataColumnsNamed(databaseName, tableName, fieldName string) []string {+ if q.gCtx.GetDatabaseMetadataFunc == nil {+ return nil+ }+ _, metadata, err := q.gCtx.GetDatabaseMetadataFunc(q.ctx, q.gCtx.InstanceID, databaseName)+ if err != nil || metadata == nil {+ return nil+ }+ schema := metadata.GetSchemaMetadata("")+ if schema == nil {+ return nil+ }+ var names []string+ for _, name := range schema.ListTableNames() {+ if name != tableName && (!q.ignoreCaseSensitive || !strings.EqualFold(name, tableName)) {+ continue+ }+ for _, column := range schema.GetTable(name).GetProto().GetColumns() {+ if strings.EqualFold(column.Name, fieldName) {+ names = append(names, column.Name)+ }+ }+ }+ return names+}++// isAggregateFunction reports whether MySQL resolves a HAVING name inside the+// function as a GROUP BY name. ANY_VALUE is not one of them.+func isAggregateFunction(name string) bool {+ switch strings.ToLower(name) {+ case "avg", "bit_and", "bit_or", "bit_xor", "count", "group_concat",+ "json_arrayagg", "json_objectagg", "max", "min", "std", "stddev",+ "stddev_pop", "stddev_samp", "sum", "var_pop", "var_samp", "variance":+ return true+ default:+ return false+ }+}++func mergeSourceColumnSetInto(dst, src base.SourceColumnSet) {+ for column := range src {+ dst[column] = true+ }+}diff --git a/backend/plugin/parser/mysql/query_span_predicate_test.go b/backend/plugin/parser/mysql/query_span_predicate_test.gonew file mode 100644index 0000000000..20d3bbc0ca--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_predicate_test.go@@ -0,0 +1,434 @@+package mysql++import (+ "context"+ "database/sql"+ "fmt"+ "slices"+ "strings"+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ gomysql "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// predicateFixtureTables is the schema the span and the engines both see.+var predicateFixtureTables = []struct {+ name string+ columns []string+ ddl string+}{+ {name: "t", columns: []string{"a", "b", "c"}, ddl: "CREATE TABLE t (a INT, b INT, c INT)"},+ {name: "t1", columns: []string{"a", "b", "c"}, ddl: "CREATE TABLE t1 (a INT, b INT, c INT)"},+ {name: "u", columns: []string{"b", "c"}, ddl: "CREATE TABLE u (b INT, c INT)"},+ {name: "j", columns: []string{"id", "doc"}, ddl: "CREATE TABLE j (id INT, doc JSON)"},+}++// predicateCase is a statement with the predicate columns its span must+// record. TestQuerySpanPredicateColumnsMatchEngines also runs it on MySQL and+// MariaDB, where the columns the engine reads must be exactly the result+// lineage, the predicate columns, and other.+type predicateCase struct {+ name string+ // statement may name the database as {db}.+ statement string+ want []string+ // results, if set, is each result column's lineage.+ results [][]string+ // other lists columns the engine reads to group or order rows, which are+ // neither result lineage nor predicates.+ other []string+ // unchecked lists predicate columns the engine compares without checking+ // the privilege to read them: those a USING or NATURAL join merges.+ unchecked []string+ // rejected means both engines refuse the statement.+ rejected bool+ // skip names an engine whose answer cannot be compared, and why.+ skip map[string]string+}++var (+ noMariaDBLateral = map[string]string{"mariadb": "MariaDB has no LATERAL"}+ mariaDBReadsUnusedCTE = map[string]string{"mariadb": "MariaDB checks privileges in a CTE it never runs"}+ noMariaDBExplainAnalyze = map[string]string{"mariadb": "MariaDB spells it ANALYZE SELECT"}+)++var predicateCases = []predicateCase{+ // WHERE.+ {name: "where column", statement: "SELECT a FROM t WHERE b = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},+ {name: "where function", statement: "SELECT a FROM t WHERE ascii(substr(c, 1, 1)) > 70", want: []string{"t.c"}},+ {name: "where row comparison", statement: "SELECT a FROM t WHERE (b, c) > (1, 2)", want: []string{"t.b", "t.c"}},+ {name: "where between in like", statement: "SELECT a FROM t WHERE b BETWEEN 1 AND c AND a IN (1, 2) AND c LIKE '1%'", want: []string{"t.a", "t.b", "t.c"}},+ {name: "where case cast", statement: "SELECT 1 FROM t WHERE CASE WHEN b > 1 THEN CAST(c AS CHAR) ELSE '' END = '1'", want: []string{"t.b", "t.c"}},+ {name: "where null-safe and regexp", statement: "SELECT 1 FROM t WHERE b <=> NULL OR c REGEXP '^1'", want: []string{"t.b", "t.c"}},+ {name: "where user variable", statement: "SELECT a FROM t WHERE b = @guess", want: []string{"t.b"}},+ {name: "where json path", statement: "SELECT id FROM j WHERE doc->>'$.k' = 'v'", want: []string{"j.doc"}, skip: map[string]string{"mariadb": "MariaDB has no ->> operator"}},+ {name: "where member of", statement: "SELECT id FROM j WHERE 1 MEMBER OF (doc->'$.arr')", want: []string{"j.doc"}, skip: map[string]string{"mariadb": "MariaDB has no MEMBER OF"}},+ {name: "where database-qualified column", statement: "SELECT t.a FROM t WHERE {db}.t.b = 1", want: []string{"t.b"}},+ {name: "where column in other case", statement: "SELECT A FROM t WHERE B = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},+ {name: "where quoted identifiers", statement: "SELECT `a` FROM `t` WHERE `b` = 1", want: []string{"t.b"}},+ {name: "count with where", statement: "SELECT count(*) FROM t WHERE c = 0", want: []string{"t.c"}, results: [][]string{{}}},+ {name: "star with where", statement: "SELECT * FROM t WHERE b = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}, {"t.b"}, {"t.c"}}},+ {name: "select list only", statement: "SELECT b FROM t", want: []string{}},+ {name: "condition in select list", statement: "SELECT CASE WHEN b = 1 THEN 1 END AS x FROM t", want: []string{}, results: [][]string{{"t.b"}}},+ {name: "limit values", statement: "SELECT a FROM t LIMIT 1 OFFSET 2", want: []string{}},++ // Joins.+ {name: "join on", statement: "SELECT t.a FROM t JOIN t1 ON t.b = t1.b", want: []string{"t.b", "t1.b"}},+ {name: "left join on", statement: "SELECT t.a FROM t LEFT JOIN t1 ON t1.a = t.a AND t1.c = 0", want: []string{"t.a", "t1.a", "t1.c"}},+ {name: "join using", statement: "SELECT t.a FROM t JOIN t1 USING (b)", want: []string{"t.b", "t1.b"}, unchecked: []string{"t.b", "t1.b"}},+ {name: "natural join", statement: "SELECT 1 FROM t NATURAL JOIN u", want: []string{"t.b", "t.c", "u.b", "u.c"}, unchecked: []string{"t.b", "t.c", "u.b", "u.c"}},+ {name: "right join using column", statement: "SELECT b FROM t RIGHT JOIN t1 USING (b) WHERE b = 99", want: []string{"t.b", "t1.b"}, results: [][]string{{"t.b", "t1.b"}}, unchecked: []string{"t.b"}},+ {name: "straight join", statement: "SELECT t.a FROM t STRAIGHT_JOIN t1 ON t.a = t1.a", want: []string{"t.a", "t1.a"}},+ {name: "nested join", statement: "SELECT 1 FROM t JOIN (t1 JOIN u ON t1.b = u.b) ON t.a = t1.a", want: []string{"t.a", "t1.a", "t1.b", "u.b"}},+ {name: "comma join", statement: "SELECT 1 FROM t, t1 WHERE t.a = t1.a", want: []string{"t.a", "t1.a"}},+ {name: "alias named like another table", statement: "SELECT x.a FROM t AS x JOIN t1 AS t ON x.a = t.a WHERE t.b = 1", want: []string{"t.a", "t1.a", "t1.b"}, results: [][]string{{"t.a"}}},+ {name: "correlated subquery in on", statement: "SELECT 1 FROM t JOIN t1 ON t1.a = (SELECT max(c) FROM u WHERE u.b = t.b)", want: []string{"t.b", "t1.a", "u.b", "u.c"}},+ {name: "cross join", statement: "SELECT t.a FROM t CROSS JOIN t1", want: []string{}, results: [][]string{{"t.a"}}},+ {name: "ambiguous column", statement: "SELECT b FROM t JOIN t1 ON t.a = t1.a", rejected: true},++ // Nested queries.+ {name: "in subquery", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM t1 WHERE b > 0)", want: []string{"t.a", "t1.a", "t1.b"}},+ {name: "correlated exists", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE t1.a = t.a AND t1.c = 0)", want: []string{"t.a", "t1.a", "t1.c"}},+ {name: "exists star is not read", statement: "SELECT a FROM t WHERE EXISTS (SELECT * FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}},+ {name: "exists output decides nothing", statement: "SELECT a FROM t WHERE EXISTS (SELECT b FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}, other: []string{"t1.b"}},+ {name: "exists wrapped in a scalar condition", statement: "SELECT 1 WHERE (SELECT EXISTS (SELECT b FROM t))", want: []string{}, other: []string{"t.b"}},+ {name: "condition in a wrapped exists", statement: "SELECT 1 WHERE (SELECT EXISTS (SELECT 1 FROM t WHERE b = 1))", want: []string{"t.b"}},+ {name: "exists in the select list keeps its lineage", statement: "SELECT EXISTS (SELECT b FROM t) AS e", want: []string{}, results: [][]string{{"t.b"}}},+ {name: "scalar subquery", statement: "SELECT a FROM t WHERE b = (SELECT max(b) FROM t1)", want: []string{"t.b", "t1.b"}},+ {name: "any subquery", statement: "SELECT a FROM t WHERE b > ANY (SELECT b FROM t1 WHERE t1.c = 0)", want: []string{"t.b", "t1.b", "t1.c"}},+ {name: "subquery name resolves inside first", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE b = 10)", want: []string{"t1.b"}},+ {name: "subquery name resolves outside", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM u WHERE u.b = a)", want: []string{"t.a", "u.b"}},+ {name: "two levels out", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE EXISTS (SELECT 1 FROM u WHERE u.b = t.b AND u.c = t1.c))", want: []string{"t.b", "t1.c", "u.b", "u.c"}},+ {name: "innermost of two levels first", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM t1 WHERE EXISTS (SELECT 1 FROM u WHERE b = 1))", want: []string{"u.b"}},+ {name: "condition of select list subquery", statement: "SELECT (SELECT count(*) FROM t1 WHERE t1.c = 0) AS k FROM t", want: []string{"t1.c"}, results: [][]string{{}}},+ {name: "select list subquery resolves inside first", statement: "SELECT (SELECT b FROM t1 LIMIT 1) AS x FROM t", want: []string{}, results: [][]string{{"t1.b"}}},+ {name: "derived table", statement: "SELECT d.a FROM (SELECT a, c FROM t WHERE b = 1) d WHERE d.c = 0", want: []string{"t.b", "t.c"}, results: [][]string{{"t.a"}}},+ {name: "derived table column list", statement: "SELECT x FROM (SELECT a, c FROM t) d (x, y) WHERE y = 0", want: []string{"t.c"}, results: [][]string{{"t.a"}}},+ {name: "lateral", statement: "SELECT l.k FROM t, LATERAL (SELECT count(*) AS k FROM t1 WHERE t1.a = t.a) l", want: []string{"t.a", "t1.a"}, results: [][]string{{}}, skip: noMariaDBLateral},+ {name: "lateral in a join sees the left operand", statement: "SELECT x.b FROM t JOIN LATERAL (SELECT t.b) AS x ON TRUE", want: []string{}, results: [][]string{{"t.b"}}, skip: noMariaDBLateral},+ {name: "lateral in a join condition", statement: "SELECT x.k FROM t JOIN LATERAL (SELECT count(*) AS k FROM t1 WHERE t1.a = t.a) AS x ON TRUE", want: []string{"t.a", "t1.a"}, results: [][]string{{}}, skip: noMariaDBLateral},+ {name: "lateral after two joined tables", statement: "SELECT x.k FROM t JOIN t1 ON t.a = t1.a JOIN LATERAL (SELECT u.c AS k FROM u WHERE u.b = t1.b) AS x ON TRUE", want: []string{"t.a", "t1.a", "t1.b", "u.b"}, results: [][]string{{"u.c"}}, skip: noMariaDBLateral},+ {name: "derived table cannot see its siblings", statement: "SELECT 1 FROM t, (SELECT a FROM t1 WHERE t1.b = t.b) d", rejected: true},+ {name: "every values row", statement: "SELECT * FROM (VALUES ROW(1), ROW((SELECT b FROM t LIMIT 1))) AS v (x)", want: []string{}, results: [][]string{{"t.b"}}, skip: map[string]string{"mariadb": "MariaDB has no VALUES ROW"}},+ {name: "window in nested condition", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM (SELECT a, ROW_NUMBER() OVER (ORDER BY b) AS r FROM t1) x WHERE r = 1)", want: []string{"t.a", "t1.a", "t1.b"}},++ // CTEs.+ {name: "cte", statement: "WITH c AS (SELECT a, b FROM t WHERE c = 0) SELECT a FROM c WHERE b = 1", want: []string{"t.b", "t.c"}, results: [][]string{{"t.a"}}},+ {name: "unread cte", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT a FROM t1", want: []string{}, results: [][]string{{"t1.a"}}, skip: mariaDBReadsUnusedCTE},+ {name: "cte read in subquery", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT 1 FROM t1 WHERE t1.a IN (SELECT a FROM c)", want: []string{"t.a", "t.b", "t1.a"}},+ {name: "cte read by cte", statement: "WITH c1 AS (SELECT a FROM t WHERE b = 1), c2 AS (SELECT a FROM c1) SELECT a FROM c2", want: []string{"t.b"}, results: [][]string{{"t.a"}}},+ {name: "cte unread by the other", statement: "WITH c1 AS (SELECT a FROM t WHERE b = 1), c2 AS (SELECT a FROM t1 WHERE c = 0) SELECT a FROM c2", want: []string{"t1.c"}, skip: mariaDBReadsUnusedCTE},+ {name: "recursive cte", statement: "WITH RECURSIVE r (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < (SELECT max(a) FROM t)) SELECT n FROM r", want: []string{"t.a"}, results: [][]string{{}}},++ // Set operations.+ {name: "union arm condition", statement: "SELECT a FROM t WHERE b = 1 UNION SELECT a FROM t1", want: []string{"t.b"}, results: [][]string{{"t.a", "t1.a"}}},+ {name: "union filters nothing", statement: "SELECT a FROM t UNION SELECT b FROM t1", want: []string{}, results: [][]string{{"t.a", "t1.b"}}},+ {name: "intersect", statement: "SELECT a FROM t INTERSECT SELECT b FROM t1", want: []string{"t.a", "t1.b"}},+ {name: "except", statement: "SELECT a FROM t EXCEPT SELECT b FROM t1", want: []string{"t.a", "t1.b"}},+ {name: "set operation limit", statement: "SELECT a FROM t UNION ALL SELECT a FROM t1 LIMIT 1", want: []string{}},+ {name: "parenthesized query", statement: "(SELECT a FROM t WHERE b = 1) ORDER BY a LIMIT 1", want: []string{"t.b"}},++ // HAVING.+ {name: "having alias of an aggregate", statement: "SELECT a, count(*) AS n FROM t GROUP BY a HAVING n > 1", want: []string{}, results: [][]string{{"t.a"}, {}}},+ {name: "having alias over a from column", statement: "SELECT c AS b FROM t HAVING b > 150", want: []string{"t.c"}, results: [][]string{{"t.c"}}},+ {name: "having group by column over an alias", statement: "SELECT c AS b FROM t GROUP BY b, c HAVING b > 150", want: []string{"t.b"}, skip: map[string]string{"mariadb": "MariaDB refuses the ambiguous name"}},+ {name: "having aggregate argument reads from", statement: "SELECT c AS b, count(*) AS k FROM t GROUP BY c HAVING max(b) > 150", want: []string{"t.b"}},+ {name: "having any_value reads the alias", statement: "SELECT c AS b FROM t GROUP BY c HAVING any_value(b) > 150", want: []string{"t.c"}, skip: map[string]string{"mariadb": "MariaDB has no ANY_VALUE"}},+ {name: "having group_concat order", statement: "SELECT a FROM t GROUP BY a HAVING group_concat(b ORDER BY c) <> ''", want: []string{"t.b", "t.c"}},+ {name: "having subquery sees from before alias", statement: "SELECT c AS b FROM u HAVING (SELECT b) > 150", want: []string{"u.b"}, results: [][]string{{"u.c"}}},+ {name: "having alias only inside aggregate", statement: "SELECT c AS x FROM u GROUP BY c HAVING max(x) > 1", want: []string{"u.c"}},+ {name: "having select item across join", statement: "SELECT t1.b FROM t JOIN t1 ON t.a = t1.a HAVING b > 5", want: []string{"t.a", "t1.a", "t1.b"}},+ {name: "having plain column", statement: "SELECT b FROM t HAVING b > 1", want: []string{"t.b"}},+ {name: "having qualified column", statement: "SELECT c AS b FROM t GROUP BY c HAVING t.c > 1", want: []string{"t.c"}},+ {name: "having subquery condition", statement: "SELECT a FROM t GROUP BY a HAVING a > (SELECT count(*) FROM t1 WHERE t1.c = 0)", want: []string{"t.a", "t1.c"}},+ {name: "having with group by position", statement: "SELECT c AS b FROM t GROUP BY 1 HAVING b > 150", want: []string{"t.c"}},+ {name: "having duplicate alias", statement: "SELECT b AS x, c AS x FROM u HAVING x > 2", rejected: true},++ // Select aliases elsewhere.+ {name: "select subquery sees an earlier alias", statement: "SELECT c AS x, (SELECT x + 1) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.c"}}},+ {name: "select subquery sees from before alias", statement: "SELECT c AS b, (SELECT b) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.b"}}},+ {name: "select subquery forward alias", statement: "SELECT (SELECT x + 1) AS s, c AS x FROM u", rejected: true},+ {name: "where subquery alias", statement: "SELECT c AS x FROM u WHERE (SELECT x) > 150", rejected: true},+ {name: "where alias", statement: "SELECT c AS x FROM u WHERE x > 1", rejected: true},+ {name: "group by subquery alias", statement: "SELECT c AS x, count(*) AS k FROM u GROUP BY (SELECT x)", want: []string{}, results: [][]string{{"u.c"}, {}}},+ {name: "group by key", statement: "SELECT count(*) FROM t GROUP BY b", want: []string{}, other: []string{"t.b"}},+ {name: "order by key", statement: "SELECT a FROM t ORDER BY c LIMIT 1", want: []string{}, other: []string{"t.c"}},+ {name: "order by subquery condition", statement: "SELECT a FROM t ORDER BY (SELECT max(b) FROM t1 WHERE t1.c = 0)", want: []string{"t1.c"}, other: []string{"t1.b"}},+ {name: "order by subquery sees from before alias", statement: "SELECT c AS b FROM u ORDER BY (SELECT b) LIMIT 1", want: []string{}, other: []string{"u.b"}},+ {name: "window keys", statement: "SELECT a, ROW_NUMBER() OVER (PARTITION BY b ORDER BY c) AS r FROM t", want: []string{}, results: [][]string{{"t.a"}, {"t.b", "t.c"}}},+ {name: "window clause subquery", statement: "SELECT a, ROW_NUMBER() OVER w AS r FROM t WINDOW w AS (ORDER BY (SELECT max(b) FROM t1 WHERE t1.c = 0))", want: []string{"t1.c"}, other: []string{"t1.b"}},++ // JSON_TABLE.+ {name: "json_table rows", statement: "SELECT jt.v FROM j, JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt", want: []string{"j.doc"}, results: [][]string{{"j.doc"}}},+ {name: "json_table condition", statement: "SELECT jt.v FROM j, JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt WHERE jt.v > 3", want: []string{"j.doc"}},++ // More shapes.+ {name: "not exists and not in", statement: "SELECT a FROM t WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE t1.c = t.c) AND a NOT IN (SELECT b FROM u)", want: []string{"t.a", "t.c", "t1.c", "u.b"}},+ {name: "row in subquery", statement: "SELECT 1 FROM t WHERE (a, b) IN (SELECT a, b FROM t1)", want: []string{"t.a", "t.b", "t1.a", "t1.b"}},+ {name: "all subquery", statement: "SELECT a FROM t WHERE b = ALL (SELECT c FROM u)", want: []string{"t.b", "u.c"}},+ {name: "union in subquery", statement: "SELECT a FROM t WHERE a IN (SELECT a FROM t1 UNION SELECT b FROM u WHERE c = 0)", want: []string{"t.a", "t1.a", "u.b", "u.c"}},+ {name: "intersect in derived table", statement: "SELECT x FROM (SELECT a AS x FROM t INTERSECT SELECT b FROM t1) d", want: []string{"t.a", "t1.b"}, results: [][]string{{"t.a", "t1.b"}}},+ {name: "boolean operators", statement: "SELECT 1 FROM t WHERE NOT b OR (c IS TRUE XOR a)", want: []string{"t.a", "t.b", "t.c"}},+ {name: "convert", statement: "SELECT 1 FROM t WHERE CONVERT(b, CHAR) = '1'", want: []string{"t.b"}},+ {name: "distinct", statement: "SELECT DISTINCT a FROM t WHERE b = 1", want: []string{"t.b"}},+ {name: "select modifiers", statement: "SELECT STRAIGHT_JOIN SQL_CALC_FOUND_ROWS t.a FROM t, t1 WHERE t.b = t1.b", want: []string{"t.b", "t1.b"}},+ {name: "cte shadows a table", statement: "WITH t AS (SELECT a FROM t1 WHERE c = 0) SELECT a FROM t WHERE a > 1", want: []string{"t1.a", "t1.c"}, results: [][]string{{"t1.a"}}},+ {name: "cte column list", statement: "WITH c (x, y) AS (SELECT a, b FROM t) SELECT x FROM c WHERE y = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}},+ {name: "cte read twice", statement: "WITH c AS (SELECT a FROM t WHERE b = 1) SELECT c1.a FROM c AS c1 JOIN c AS c2 ON c1.a = c2.a", want: []string{"t.a", "t.b"}},+ {name: "json_table joined on", statement: "SELECT jt.v FROM j JOIN JSON_TABLE(j.doc, '$[*]' COLUMNS (v INT PATH '$')) jt ON jt.v = j.id", want: []string{"j.doc", "j.id"}},+ {name: "having alias in other case", statement: "SELECT c AS X FROM u HAVING x > 1", want: []string{"u.c"}},+ {name: "having quoted alias", statement: "SELECT c AS `my col` FROM u HAVING `my col` > 1", want: []string{"u.c"}},+ {name: "having alias in case", statement: "SELECT c AS b FROM u HAVING CASE WHEN b > 1 THEN 1 ELSE 0 END = 1", want: []string{"u.c"}},+ {name: "having count distinct", statement: "SELECT a FROM t GROUP BY a HAVING count(DISTINCT b) > 1", want: []string{"t.b"}},+ {name: "having with rollup", statement: "SELECT a, count(*) AS n FROM t GROUP BY a WITH ROLLUP HAVING a IS NULL", want: []string{"t.a"}},+ {name: "having alias of a joined column", statement: "SELECT t1.c AS b FROM t JOIN t1 ON t.a = t1.a HAVING b > 1", want: []string{"t.a", "t1.a", "t1.c"}},+ {name: "order by alias", statement: "SELECT c AS b FROM u ORDER BY b LIMIT 1", want: []string{}, results: [][]string{{"u.c"}}},+ {name: "order by expression reads from first", statement: "SELECT c AS b FROM u ORDER BY b + 0 LIMIT 1", want: []string{}, other: []string{"u.b"}},+ {name: "group by reads from first", statement: "SELECT max(b) AS c FROM u GROUP BY c", want: []string{}, results: [][]string{{"u.b"}}, other: []string{"u.c"}},+ {name: "exists over a qualified star", statement: "SELECT a FROM t WHERE EXISTS (SELECT t1.* FROM t1 WHERE t1.a = t.a)", want: []string{"t.a", "t1.a"}},+ {name: "derived table in a subquery sees the outer query", statement: "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM (SELECT b FROM t1 WHERE t1.a = t.a) d)", want: []string{"t.a", "t1.a"}, skip: map[string]string{"mariadb": "MariaDB refuses an outer reference in a derived table"}},+ {name: "subquery two levels down sees an alias", statement: "SELECT c AS x, (SELECT (SELECT x + 1)) AS s FROM u", want: []string{}, results: [][]string{{"u.c"}, {"u.c"}}},++ // EXPLAIN ANALYZE runs the query and reports the rows each step returns.+ {name: "explain analyze condition", statement: "EXPLAIN ANALYZE SELECT 1 FROM t WHERE b = 123", want: []string{"t.b"}, skip: noMariaDBExplainAnalyze},+ {name: "explain analyze having alias", statement: "EXPLAIN ANALYZE SELECT c AS b FROM t HAVING b > 150", want: []string{"t.c"}, skip: noMariaDBExplainAnalyze},+ {name: "explain analyze nested condition", statement: "EXPLAIN ANALYZE SELECT a FROM t WHERE a IN (SELECT a FROM t1 WHERE c = 0)", want: []string{"t.a", "t1.a", "t1.c"}, skip: noMariaDBExplainAnalyze},++ // A view's own conditions are not the caller's; its columns trace to the+ // table. The engines check a view's privileges on the view, so they skip it.+ {name: "view column", statement: "SELECT a FROM v WHERE vb = 1", want: []string{"t.b"}, results: [][]string{{"t.a"}}, skip: map[string]string{"mysql": "privileges are checked on the view", "mariadb": "privileges are checked on the view"}},+}++func TestQuerySpanPredicateColumns(t *testing.T) {+ for _, tc := range predicateCases {+ t.Run(tc.name, func(t *testing.T) {+ span, err := predicateCaseSpan("db", strings.ReplaceAll(tc.statement, "{db}", "db"))+ if tc.rejected {+ return+ }+ require.NoError(t, err)+ require.Nil(t, span.NotFoundError)+ require.Equal(t, sortedOrEmpty(tc.want), columnNames(span.PredicateColumns))+ if tc.results != nil {+ var results [][]string+ for _, result := range span.Results {+ results = append(results, columnNames(result.SourceColumns))+ }+ want := make([][]string, 0, len(tc.results))+ for _, columns := range tc.results {+ want = append(want, sortedOrEmpty(columns))+ }+ require.Equal(t, want, results)+ }+ })+ }+}++func TestQuerySpanPredicateColumnsFallBackByName(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want []string+ }{+ {"unknown table in a subquery", "SELECT a FROM t WHERE a IN (SELECT x FROM missing)", []string{"t.a"}},+ {"unknown table in an exists", "SELECT a FROM t WHERE EXISTS (SELECT 1 FROM missing WHERE missing.b = t.b)", []string{"t.b"}},+ {"unknown column", "SELECT a FROM t WHERE b = 1 AND missing = 2", []string{"t.b"}},+ {"unknown table in an order by subquery", "SELECT a FROM t ORDER BY (SELECT 1 FROM missing WHERE c = 0)", []string{"t.c"}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ span, err := predicateCaseSpan("db", tc.statement)+ require.NoError(t, err)+ require.Nil(t, span.NotFoundError)+ require.Equal(t, tc.want, columnNames(span.PredicateColumns))+ })+ }+}++// TestQuerySpanPredicateColumnsMatchEngines runs each case on a user denied+// one fixture column at a time: the engine refuses exactly the statements+// that read the column, whichever name or clause the statement reads it by.+func TestQuerySpanPredicateColumnsMatchEngines(t *testing.T) {+ for _, engine := range []struct {+ name string+ newDatabase func(testing.TB) (*testcontainer.Container, string)+ }{+ {name: "mysql", newDatabase: testcontainer.NewMySQLDatabase},+ {name: "mariadb", newDatabase: testcontainer.NewMariaDBDatabase},+ } {+ t.Run(engine.name, func(t *testing.T) {+ container, database := engine.newDatabase(t)+ oracle := newColumnReadOracle(t, container, database)+ for _, tc := range predicateCases {+ t.Run(tc.name, func(t *testing.T) {+ if reason, ok := tc.skip[engine.name]; ok {+ t.Skip(reason)+ }+ statement := strings.ReplaceAll(tc.statement, "{db}", database)+ read, err := oracle.columnsRead(statement)+ if tc.rejected {+ require.ErrorIs(t, err, errRejected)+ return+ }+ require.NoError(t, err)++ span, err := predicateCaseSpan(database, statement)+ require.NoError(t, err)+ seen := make(base.SourceColumnSet)+ for _, result := range span.Results {+ mergeSourceColumnSetInto(seen, result.SourceColumns)+ }+ mergeSourceColumnSetInto(seen, span.PredicateColumns)+ got := append(columnNames(seen), tc.other...)+ got = slices.DeleteFunc(got, func(column string) bool { return slices.Contains(tc.unchecked, column) })+ slices.Sort(got)+ require.Equal(t, read, slices.Compact(got))+ })+ }+ })+ }+}++var errRejected = errors.New("the engine refuses the statement")++type columnReadOracle struct {+ root *sql.DB+ // deniedOne maps each fixture column to a session that may read every+ // fixture column but that one.+ deniedOne map[string]*sql.DB+}++func newColumnReadOracle(t *testing.T, container *testcontainer.Container, database string) *columnReadOracle {+ t.Helper()+ open := func(user, password string) *sql.DB {+ db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", user, password, container.GetHost(), container.GetPort(), database))+ require.NoError(t, err)+ db.SetMaxOpenConns(1)+ t.Cleanup(func() { require.NoError(t, db.Close()) })+ return db+ }+ oracle := &columnReadOracle{+ root: open(container.GetUsername(), container.GetPassword()),+ deniedOne: make(map[string]*sql.DB),+ }+ for _, table := range predicateFixtureTables {+ _, err := oracle.root.Exec(table.ddl)+ require.NoError(t, err)+ }+ _, err := oracle.root.Exec("CREATE VIEW v AS SELECT a, b AS vb FROM t WHERE c > 0")+ require.NoError(t, err)++ i := 0+ for _, denied := range predicateFixtureTables {+ for _, deniedColumn := range denied.columns {+ i+++ // Users are server-wide, so they carry the database's name.+ user := fmt.Sprintf("%s_%d", database, i)+ _, err := oracle.root.Exec(fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY 'pw'", user))+ require.NoError(t, err)+ t.Cleanup(func() {+ _, err := container.GetDB().Exec(fmt.Sprintf("DROP USER '%s'@'%%'", user))+ require.NoError(t, err)+ })+ for _, table := range predicateFixtureTables {+ grant := fmt.Sprintf("GRANT SELECT ON `%s`.`%s` TO '%s'@'%%'", database, table.name, user)+ if table.name == denied.name {+ var allowed []string+ for _, column := range table.columns {+ if column != deniedColumn {+ allowed = append(allowed, "`"+column+"`")+ }+ }+ grant = fmt.Sprintf("GRANT SELECT (%s) ON `%s`.`%s` TO '%s'@'%%'", strings.Join(allowed, ", "), database, table.name, user)+ }+ _, err := oracle.root.Exec(grant)+ require.NoError(t, err)+ }+ oracle.deniedOne[denied.name+"."+deniedColumn] = open(user, "pw")+ }+ }+ return oracle+}++// columnsRead returns the fixture columns the engine reads to run statement,+// or errRejected if it refuses the statement outright.+func (o *columnReadOracle) columnsRead(statement string) ([]string, error) {+ if err := drain(o.root, statement); err != nil {+ return nil, errors.Wrap(errRejected, err.Error())+ }+ read := []string{}+ for column, db := range o.deniedOne {+ err := drain(db, statement)+ if err == nil {+ continue+ }+ var mysqlErr *gomysql.MySQLError+ // 1142 is a table-level refusal, which * over the table raises.+ if errors.As(err, &mysqlErr) && (mysqlErr.Number == 1142 || mysqlErr.Number == 1143) {+ read = append(read, column)+ continue+ }+ return nil, errors.Wrapf(err, "denied %s", column)+ }+ slices.Sort(read)+ return read, nil+}++func drain(db *sql.DB, statement string) error {+ rows, err := db.Query(statement)+ if err != nil {+ return err+ }+ defer rows.Close()+ for rows.Next() {+ }+ return rows.Err()+}++func predicateCaseSpan(database, statement string) (*base.QuerySpan, error) {+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{predicateFixtureMetadata(database)})+ return GetQuerySpan(context.Background(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ Engine: storepb.Engine_MYSQL,+ }, base.Statement{Text: statement}, database, "", false)+}++func predicateFixtureMetadata(database string) *metadatapb.DatabaseSchemaMetadata {+ schema := &metadatapb.SchemaMetadata{+ Views: []*metadatapb.ViewMetadata{{Name: "v", Definition: "SELECT a, b AS vb FROM t WHERE c > 0"}},+ }+ for _, table := range predicateFixtureTables {+ metadata := &metadatapb.TableMetadata{Name: table.name}+ for _, column := range table.columns {+ metadata.Columns = append(metadata.Columns, &metadatapb.ColumnMetadata{Name: column})+ }+ schema.Tables = append(schema.Tables, metadata)+ }+ return &metadatapb.DatabaseSchemaMetadata{Name: database, Schemas: []*metadatapb.SchemaMetadata{schema}}+}++// columnNames returns the table.column names in columns, sorted, skipping+// table-level entries.+func columnNames(columns base.SourceColumnSet) []string {+ names := []string{}+ for column := range columns {+ if column.Column != "" {+ names = append(names, column.Table+"."+column.Column)+ }+ }+ slices.Sort(names)+ return names+}++func sortedOrEmpty(names []string) []string {+ sorted := append([]string{}, names...)+ slices.Sort(sorted)+ return sorted+}diff --git a/backend/plugin/parser/mysql/test-data/query-span/standard.yaml b/backend/plugin/parser/mysql/test-data/query-span/standard.yamlindex 789a4d03f2..be674adb9e 100644--- a/backend/plugin/parser/mysql/test-data/query-span/standard.yaml+++ b/backend/plugin/parser/mysql/test-data/query-span/standard.yaml@@ -73,7 +73,12 @@schema: ""table: productscolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: products+ column: product_info- description: Test For json tablestatement: |-SELECT *@@ -154,7 +159,12 @@schema: ""table: productscolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: products+ column: product_info- description: Test For alias.* with joinstatement: SELECT t1.*, t2.* FROM t t1 JOIN (select * from t) t2 ON t1.a = t2.a;defaultDatabase: db@@ -206,7 +216,12 @@schema: ""table: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: t+ column: a- description: Test For Explain statementsstatement: EXPLAIN SELECT 1;defaultDatabase: db@@ -635,7 +650,12 @@schema: ""table: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: t+ column: a- description: Correlated subquery in the target field clause.statement: SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total_paintings FROM galleries g;defaultDatabase: db@@ -707,7 +727,17 @@schema: ""table: paintingscolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: galleries+ column: id+ - server: ""+ database: db+ schema: ""+ table: paintings+ column: gallery_id- description: Test for functionsstatement: SELECT max(a), a-b AS c1, a=b AS c2, a>b, b in (a, c, d) from (SELECT * FROM t) result;defaultDatabase: db@@ -1025,7 +1055,12 @@schema: ""table: tcolumn: ""- predicatecolumns: []+ predicatecolumns:+ - server: ""+ database: db+ schema: ""+ table: t+ column: a- description: Simple CTEstatement: WITH t1 AS (SELECT * FROM t) SELECT * FROM t1;defaultDatabase: db