Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore(mysql): migrate query span to omni (#20141)
rebelicefc2c27b3bbd8
diff --git a/backend/plugin/parser/mysql/query.go b/backend/plugin/parser/mysql/query.goindex a67f784e21..b4e4a3c0c5 100644--- a/backend/plugin/parser/mysql/query.go+++ b/backend/plugin/parser/mysql/query.go@@ -19,7 +19,7 @@ func init() {func validateQuery(statement string) (bool, bool, error) {stmts, err := ParseMySQLOmni(statement)if err != nil {- return false, false, err+ return false, false, convertOmniError(err, base.Statement{Text: statement})}hasExecute := falsediff --git a/backend/plugin/parser/mysql/query_span.go b/backend/plugin/parser/mysql/query_span.goindex 666bf63b05..dc0624252a 100644--- a/backend/plugin/parser/mysql/query_span.go+++ b/backend/plugin/parser/mysql/query_span.go@@ -24,7 +24,7 @@ func GetQuerySpan(q := newQuerySpanExtractor(database, gCtx, ignoreCaseSensitive)querySpan, err := q.getQuerySpan(ctx, stmt.Text)if err != nil {- return nil, err+ return nil, convertOmniError(err, stmt)}return querySpan, nil}diff --git a/backend/plugin/parser/mysql/query_span_extractor.go b/backend/plugin/parser/mysql/query_span_extractor.goindex 19a4abbf72..0b902424d1 100644--- a/backend/plugin/parser/mysql/query_span_extractor.go+++ b/backend/plugin/parser/mysql/query_span_extractor.go@@ -2,20 +2,17 @@ package mysqlimport ("context"- "slices""strings"- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/mysql""github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/store/model"-"github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model")-// querySpanExtractor is the extractor to extract the query span from a single statement.+// querySpanExtractor contains shared query-span state and metadata helpers.+// MySQL query-span extraction itself is omni-backed in query_span_extractor_omni.go.type querySpanExtractor struct {ctx context.ContextdefaultDatabase string@@ -24,27 +21,21 @@ type querySpanExtractor struct {ignoreCaseSensitive bool- // ctes is the common table expressions, which is used to record the cte schema.- // It should be reset to original state while quit the nested cte. For example:- // WITH cte_1 AS (WITH cte_2 AS (SELECT * FROM t) SELECT * FROM cte_2) SELECT * FROM cte_2;- // MySQL will throw a runtime error: (1146, "Table 'junk.cte_2' doesn't exist"), the `junk` is the database name.+ // ctes records common table expressions visible to the current select scope.ctes []*base.PseudoTable- // outerTableSources is the table sources from the outer query span.- // it's used to resolve the column name in the correlated sub-query.+ // outerTableSources resolves correlated subquery column references.outerTableSources []base.TableSource- // tableSourceFrom is the table sources from the from clause.+ // tableSourceFrom is the set of table sources visible from the current FROM clause.tableSourceFrom []base.TableSource- // priorTableInFrom is the table sources from the from clause before the current table source.- // It's used to resolve the column name in JSON_TABLE functions.+ // priorTableInFrom resolves JSON_TABLE document expressions against preceding FROM items.priorTableInFrom []base.TableSourceviewResolutionStack map[string]bool}-// newQuerySpanExtractor creates a new query span extractor, the databaseMetadata and the ast are in the read guard.func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *querySpanExtractor {return &querySpanExtractor{defaultDatabase: defaultDatabase,@@ -54,586 +45,7 @@ func newQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContext}func (q *querySpanExtractor) getQuerySpan(ctx context.Context, stmt string) (*base.QuerySpan, error) {- parseResults, err := ParseMySQL(stmt)- if err != nil {- return nil, err- }- if len(parseResults) == 0 {- return &base.QuerySpan{- SourceColumns: base.SourceColumnSet{},- Results: []base.QuerySpanResult{},- }, nil- }- if len(parseResults) != 1 {- return nil, errors.Errorf("expecting only one statement to get query span, but got %d", len(parseResults))- }- tree := parseResults[0].Tree-- accessTables := getAccessTables(q.defaultDatabase, tree)- // We do not support simultaneous access to the system table and the user table- // because we do not synchronize the schema of the system table.- // This causes an error (NOT_FOUND) when using querySpanExtractor.findTableSchema.- // As a result, we exclude getting query span results for accessing only the system table.- allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)- if mixed {- return nil, base.MixUserSystemTablesError- }-- queryTypeListener := &queryTypeListener{- allSystems: allSystems,- result: base.QueryTypeUnknown,- }- antlr.ParseTreeWalkerDefault.Walk(queryTypeListener, tree)-- if queryTypeListener.result != base.Select {- return &base.QuerySpan{- Type: queryTypeListener.result,- SourceColumns: base.SourceColumnSet{},- Results: []base.QuerySpanResult{},- }, nil- }-- // For Explain Analyze SELECT, we determine the query type and return the access tables.- if queryTypeListener.isExplainAnalyze {- return &base.QuerySpan{- Type: queryTypeListener.result,- SourceColumns: accessTables,- Results: []base.QuerySpanResult{},- }, nil- }-- q.ctx = ctx- // We assume the caller had handled the statement type case,- // so we only need to handle the determined statement type here.- // In order to decrease the maintenance cost, we use listener to handle- // the select statement precisely instead of using type switch.- listener := newSelectOnlyListener(q)- antlr.ParseTreeWalkerDefault.Walk(listener, tree)- err = listener.err- if err != nil {- var resourceNotFound *base.ResourceNotFoundError- if errors.As(err, &resourceNotFound) {- return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessTables,- Results: []base.QuerySpanResult{},- NotFoundError: resourceNotFound,- }, nil- }-- return nil, err- }- return &base.QuerySpan{- Type: base.Select,- SourceColumns: accessTables,- Results: listener.querySpan.Results,- }, nil-}--func (q *querySpanExtractor) extractContext(ctx antlr.ParserRuleContext) (*base.PseudoTable, error) {- if ctx == nil {- return nil, nil- }-- switch ctx := ctx.(type) {- case parser.ISelectStatementContext:- return q.extractSelectStatement(ctx)- default:- return nil, nil- }-}--// extractSelectStatement extracts the table source from the select statement.-// It regards the result of the select statement as the pseudo table.-func (q *querySpanExtractor) extractSelectStatement(ctx parser.ISelectStatementContext) (*base.PseudoTable, error) {- if ctx == nil {- return base.NewPseudoTable("", []base.QuerySpanResult{}), nil- }-- switch {- case ctx.QueryExpression() != nil:- return q.extractQueryExpression(ctx.QueryExpression())- case ctx.QueryExpressionParens() != nil:- return q.extractQueryExpressionParens(ctx.QueryExpressionParens())- case ctx.SelectStatementWithInto() != nil:- return nil, errors.New("meet unsupported select statement with into")- default:- return nil, errors.New("unexpected select statement")- }-}--func (q *querySpanExtractor) extractQueryExpression(ctx parser.IQueryExpressionContext) (*base.PseudoTable, error) {- if ctx == nil {- return base.NewPseudoTable("", []base.QuerySpanResult{}), nil- }-- if ctx.WithClause() != nil {- originalCtesLength := len(q.ctes)- defer func() {- q.ctes = q.ctes[:originalCtesLength]- }()- recursive := ctx.WithClause().RECURSIVE_SYMBOL() != nil- for _, cte := range ctx.WithClause().AllCommonTableExpression() {- cteTable, err := q.extractCommonTableExpression(cte, recursive)- if err != nil {- return nil, err- }- q.ctes = append(q.ctes, cteTable)- }- }-- switch {- case ctx.QueryExpressionParens() != nil:- return q.extractQueryExpressionParens(ctx.QueryExpressionParens())- case ctx.QueryExpressionBody() != nil:- return q.extractQueryExpressionBody(ctx.QueryExpressionBody())- default:- panic("unreachable")- }-}--func (q *querySpanExtractor) extractQueryExpressionParens(ctx parser.IQueryExpressionParensContext) (*base.PseudoTable, error) {- if ctx == nil {- return nil, nil- }-- switch {- case ctx.QueryExpression() != nil:- return q.extractQueryExpression(ctx.QueryExpression())- case ctx.QueryExpressionParens() != nil:- return q.extractQueryExpressionParens(ctx.QueryExpressionParens())- default:- return nil, nil- }-}--func (q *querySpanExtractor) extractQueryExpressionBody(ctx parser.IQueryExpressionBodyContext) (*base.PseudoTable, error) {- var result base.TableSource- unionNum := 0- for _, child := range ctx.GetChildren() {- switch child := child.(type) {- case *parser.QueryPrimaryContext:- unionNum++- ts, err := q.extractQueryPrimary(child)- if err != nil {- return nil, err- }- if result == nil {- result = ts- } else {- if len(result.GetQuerySpanResult()) != len(ts.GetQuerySpanResult()) {- return nil, errors.Errorf("MySQL %d UNION operator left has %d fields, right has %d fields", unionNum, len(result.GetQuerySpanResult()), len(ts.GetQuerySpanResult()))- }- // The UNION operator will merge the result columns of the left and right table sources.- newQuerySpanResults := make([]base.QuerySpanResult, 0, len(result.GetQuerySpanResult()))- resultQuerySpanResult := result.GetQuerySpanResult()- for i := range ts.GetQuerySpanResult() {- newSourceColumnSet, _ := base.MergeSourceColumnSet(resultQuerySpanResult[i].SourceColumns, ts.GetQuerySpanResult()[i].SourceColumns)- newQuerySpanResults = append(newQuerySpanResults, base.QuerySpanResult{- Name: resultQuerySpanResult[i].Name,- SourceColumns: newSourceColumnSet,- })- }- result = &base.PseudoTable{- Name: "",- Columns: newQuerySpanResults,- }- }- case *parser.QueryExpressionParensContext:- ts, err := q.extractQueryExpressionParens(child)- if err != nil {- return nil, err- }- if result == nil {- result = ts- } else {- if len(result.GetQuerySpanResult()) != len(ts.GetQuerySpanResult()) {- return nil, errors.Errorf("MySQL %d UNION operator left has %d fields, right has %d fields", unionNum, len(result.GetQuerySpanResult()), len(ts.GetQuerySpanResult()))- }- // The UNION operator will merge the result columns of the left and right table sources.- newQuerySpanResults := make([]base.QuerySpanResult, 0, len(result.GetQuerySpanResult()))- resultQuerySpanResult := result.GetQuerySpanResult()- for i := range ts.GetQuerySpanResult() {- newSourceColumnSet, _ := base.MergeSourceColumnSet(resultQuerySpanResult[i].SourceColumns, ts.GetQuerySpanResult()[i].SourceColumns)- newQuerySpanResults = append(newQuerySpanResults, base.QuerySpanResult{- Name: resultQuerySpanResult[i].Name,- SourceColumns: newSourceColumnSet,- })- }- result = &base.PseudoTable{- Name: "",- Columns: newQuerySpanResults,- }- }- default:- }- }-- return &base.PseudoTable{- Name: "",- Columns: result.GetQuerySpanResult(),- }, nil-}--func (q *querySpanExtractor) extractQueryPrimary(ctx parser.IQueryPrimaryContext) (base.TableSource, error) {- switch {- case ctx.QuerySpecification() != nil:- return q.extractQuerySpecification(ctx.QuerySpecification())- case ctx.TableValueConstructor() != nil:- return q.extractTableValueConstructor(ctx.TableValueConstructor())- case ctx.ExplicitTable() != nil:- return q.extractExplicitTable(ctx.ExplicitTable())- default:- panic("unreachable")- }-}--func (q *querySpanExtractor) extractExplicitTable(ctx parser.IExplicitTableContext) (base.TableSource, error) {- if ctx == nil {- return nil, nil- }-- databaseName, tableName := NormalizeMySQLTableRef(ctx.TableRef())- tableSource, err := q.findTableSchema(databaseName, tableName)- if err != nil {- return nil, err- }-- return tableSource, nil-}--func (q *querySpanExtractor) extractTableValueConstructor(ctx parser.ITableValueConstructorContext) (*base.PseudoTable, error) {- if ctx == nil {- return nil, nil- }-- firstRow := ctx.RowValueExplicit(0)- if firstRow == nil {- panic("unreachable zero row in table value constructor")- }-- values := firstRow.Values()- if values == nil {- return &base.PseudoTable{- Name: "",- Columns: []base.QuerySpanResult{},- }, nil- }-- var columns []base.QuerySpanResult-- for _, child := range values.GetChildren() {- switch child := child.(type) {- case *parser.ExprContext:- _, sourceColumns, isPlain, err := q.extractSourceColumnSetFromExpr(child)- if err != nil {- return nil, err- }- columns = append(columns, base.QuerySpanResult{- Name: child.GetParser().GetTokenStream().GetTextFromRuleContext(child),- SourceColumns: sourceColumns,- IsPlainField: isPlain,- })- case antlr.TerminalNode:- if child.GetSymbol().GetTokenType() == parser.MySQLParserDEFAULT_SYMBOL {- columns = append(columns, base.QuerySpanResult{- Name: "DEFAULT",- SourceColumns: make(base.SourceColumnSet),- IsPlainField: true,- })- }- default:- }- }-- return &base.PseudoTable{- Name: "",- Columns: columns,- }, nil-}--func (q *querySpanExtractor) extractQuerySpecification(ctx parser.IQuerySpecificationContext) (*base.PseudoTable, error) {- var fromSources []base.TableSource- var err error- if ctx.FromClause() != nil {- originalLength := len(q.tableSourceFrom)- defer func() {- q.tableSourceFrom = q.tableSourceFrom[:originalLength]- }()- originalPriorLength := len(q.priorTableInFrom)- defer func() {- q.priorTableInFrom = q.priorTableInFrom[:originalPriorLength]- }()- fromSources, err = q.extractTableSourcesFromFromClause(ctx.FromClause())- if err != nil {- return nil, err- }- q.tableSourceFrom = append(q.tableSourceFrom, fromSources...)- }-- querySpanResult, err := q.extractSelectItemList(ctx.SelectItemList(), fromSources)- if err != nil {- return nil, err- }- return &base.PseudoTable{- Name: "",- Columns: querySpanResult,- }, nil-}--func (q *querySpanExtractor) extractSelectItemList(ctx parser.ISelectItemListContext, fromSpanResult []base.TableSource) ([]base.QuerySpanResult, error) {- if ctx == nil {- return nil, nil- }-- var result []base.QuerySpanResult-- if ctx.MULT_OPERATOR() != nil {- for _, fromSpan := range fromSpanResult {- result = append(result, fromSpan.GetQuerySpanResult()...)- }- }-- for _, selectItem := range ctx.AllSelectItem() {- spanResult, err := q.extractSelectItem(selectItem)- if err != nil {- return nil, err- }- result = append(result, spanResult...)- }-- return result, nil-}--func (q *querySpanExtractor) extractSelectItem(ctx parser.ISelectItemContext) ([]base.QuerySpanResult, error) {- if ctx == nil {- return nil, nil- }-- switch {- case ctx.TableWild() != nil:- return q.extractTableWild(ctx.TableWild())- case ctx.Expr() != nil:- fieldName, sourceColumns, isPlain, err := q.extractSourceColumnSetFromExpr(ctx.Expr())- if err != nil {- return nil, err- }- if ctx.SelectAlias() != nil {- fieldName = NormalizeMySQLSelectAlias(ctx.SelectAlias())- } else if fieldName == "" {- fieldName = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- }-- return []base.QuerySpanResult{- {- Name: fieldName,- SourceColumns: sourceColumns,- IsPlainField: isPlain,- },- }, nil- default:- panic("unreachable")- }-}--func (q *querySpanExtractor) extractSourceColumnSetFromExpr(ctx antlr.ParserRuleContext) (string, base.SourceColumnSet, bool, error) {- if ctx == nil {- return "", make(base.SourceColumnSet), true, nil- }-- // The closure of expr rules.- switch ctx := ctx.(type) {- case parser.ISubqueryContext:- baseSet := make(base.SourceColumnSet)- // Subquery in SELECT fields is special.- // It can be the non-associated or associated subquery.- // For associated subquery, we should set the fromFieldList as the outerSchemaInfo.- // So that the subquery can access the outer schema.- // The reason for new extractor is that we still need the current fromFieldList, overriding it is not expected.- subqueryExtractor := &querySpanExtractor{- ctx: q.ctx,- // The defaultDatabase is the same as the outer query.- defaultDatabase: q.defaultDatabase,- gCtx: q.gCtx,- ctes: q.ctes,- outerTableSources: append(q.outerTableSources, q.tableSourceFrom...),- tableSourceFrom: []base.TableSource{},- viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),- }- tableSource, err := subqueryExtractor.extractSubquery(ctx)- if err != nil {- return "", nil, false, err- }- spanResult := tableSource.GetQuerySpanResult()- isPlain := false- if len(spanResult) == 1 {- isPlain = spanResult[0].IsPlainField- }- for _, field := range spanResult {- baseSet, _ = base.MergeSourceColumnSet(field.SourceColumns, field.SourceColumns)- }- return "", baseSet, isPlain, nil- case parser.IColumnRefContext:- databaseName, tableName, fieldName := NormalizeMySQLFieldIdentifier(ctx.FieldIdentifier())- sourceColumnSet, err := q.getFieldColumnSource(databaseName, tableName, fieldName)- if err != nil {- return "", nil, false, err- }- return fieldName, sourceColumnSet, true, nil- }-- var list []antlr.ParserRuleContext- for _, child := range ctx.GetChildren() {- if child, ok := child.(antlr.ParserRuleContext); ok {- list = append(list, child)- }- }-- fieldName, sourceColumnSet, plain, err := q.extractSourceColumnSetFromExprList(list)- if err != nil {- return "", nil, false, err- }- if len(ctx.GetChildren()) > 1 {- fieldName = ""- }- return fieldName, sourceColumnSet, plain && len(ctx.GetChildren()) < 2, nil-}--func (q *querySpanExtractor) extractSourceColumnSetFromExprList(ctxs []antlr.ParserRuleContext) (string, base.SourceColumnSet, bool, error) {- baseSet := make(base.SourceColumnSet)- var fieldName string- var set base.SourceColumnSet- var err error- plain := true- for _, ctx := range ctxs {- var isPlain bool- fieldName, set, isPlain, err = q.extractSourceColumnSetFromExpr(ctx)- if err != nil {- return "", nil, false, err- }- plain = plain && isPlain- baseSet, _ = base.MergeSourceColumnSet(baseSet, set)- }-- return fieldName, baseSet, plain, nil-}--func (q *querySpanExtractor) extractTableWild(ctx parser.ITableWildContext) ([]base.QuerySpanResult, error) {- if ctx == nil {- return nil, nil- }-- var databaseName, tableName string- if len(ctx.AllIdentifier()) == 2 {- databaseName = NormalizeMySQLIdentifier(ctx.Identifier(0))- tableName = NormalizeMySQLIdentifier(ctx.Identifier(1))- } else {- tableName = NormalizeMySQLIdentifier(ctx.Identifier(0))- }- querySpanResults, ok := q.getAllTableColumnSources(databaseName, tableName)- if !ok {- return nil, &base.ResourceNotFoundError{- Err: errors.Errorf("failed to find table to calculate asterisk"),- Database: &databaseName,- Table: &tableName,- }- }- return querySpanResults, nil-}--// extractTableSourcesFromFromClause extracts the table sources from the from clause.-// The result can be empty while the from clause is dual.-func (q *querySpanExtractor) extractTableSourcesFromFromClause(ctx parser.IFromClauseContext) ([]base.TableSource, error) {- // DUAL is purely for the convenience of people who require that all SELECT statements should have FROM and possibly other clauses.- // MySQL may ignore the clauses. MySQL does not require FROM DUAL if no tables are referenced.- if ctx.DUAL_SYMBOL() != nil {- return []base.TableSource{}, nil- }-- return q.extractTableReferenceList(ctx.TableReferenceList())-}--func (q *querySpanExtractor) extractTableReferenceList(ctx parser.ITableReferenceListContext) ([]base.TableSource, error) {- var result []base.TableSource- for _, tableReference := range ctx.AllTableReference() {- tableResource, err := q.extractTableReference(tableReference)- if err != nil {- return nil, err- }- q.priorTableInFrom = append(q.priorTableInFrom, tableResource)- result = append(result, tableResource)- }-- return result, nil-}--func (q *querySpanExtractor) extractTableReference(ctx parser.ITableReferenceContext) (base.TableSource, error) {- if ctx.TableFactor() == nil {- return nil, errors.Errorf("MySQL table reference should have table factor")- }-- tableSource, err := q.extractTableFactor(ctx.TableFactor())- if err != nil {- return nil, err- }-- if len(ctx.AllJoinedTable()) == 0 {- return tableSource, nil- }-- q.tableSourceFrom = append(q.tableSourceFrom, tableSource)-- for _, joinedTable := range ctx.AllJoinedTable() {- tableSource, err = q.extractJoinedTable(tableSource, joinedTable)- if err != nil {- return nil, err- }- }-- return tableSource, nil-}--// extractJoinedTable extracts the joined table from the left and right table sources.-func (q *querySpanExtractor) extractJoinedTable(l base.TableSource, r parser.IJoinedTableContext) (base.TableSource, error) {- rightTableSource, err := q.extractTableReference(r.TableReference())- if err != nil {- return nil, err- }- q.tableSourceFrom = append(q.tableSourceFrom, rightTableSource)-- tp := Join-- if v := r.InnerJoinType(); v != nil {- if v.INNER_SYMBOL() != nil {- tp = InnerJoin- } else if v.CROSS_SYMBOL() != nil {- tp = CrossJoin- } else if v.STRAIGHT_JOIN_SYMBOL() != nil {- tp = StraightJoin- }- } else if v := r.OuterJoinType(); v != nil {- if v.LEFT_SYMBOL() != nil {- tp = LeftOuterJoin- } else if v.RIGHT_SYMBOL() != nil {- tp = RightOuterJoin- }- } else if v := r.NaturalJoinType(); v != nil {- tp = NaturalInnerJoin- if v.LEFT_SYMBOL() != nil {- tp = NaturalLeftOuterJoin- } else if v.RIGHT_SYMBOL() != nil {- tp = NaturalRightOuterJoin- }- }-- var usingIdentifiers []string- if r.IdentifierListWithParentheses() != nil && r.IdentifierListWithParentheses().IdentifierList() != nil {- usingIdentifiers = NormalizeMySQLIdentifierList(r.IdentifierListWithParentheses().IdentifierList())- }-- joinedTableSource := joinTableSources(l, rightTableSource, tp, usingIdentifiers)-- return joinedTableSource, nil+ return (&omniQuerySpanExtractor{querySpanExtractor: q}).getOmniQuerySpan(ctx, stmt)}type joinType int@@ -650,21 +62,14 @@ const (NaturalRightOuterJoin)-// joinTableSources joins the left and right table sources with the join type.func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.TableSource {switch tp {case Join, InnerJoin, CrossJoin, StraightJoin, LeftOuterJoin, RightOuterJoin:var columns []base.QuerySpanResult- // In MySQL, JOIN, CROSS JOIN, and INNER JOIN are syntactic equivalents (they can replace each other).- leftFieldsMap := make(map[string]bool)- for _, field := range l.GetQuerySpanResult() {- leftFieldsMap[strings.ToLower(field.Name)] = true- }rightFieldsMap := make(map[string]bool)for _, field := range r.GetQuerySpanResult() {rightFieldsMap[strings.ToLower(field.Name)] = true}- // Using will merge the result columns of the left and right table sources.lowercaseUsingFields := make(map[string]bool)for _, field := range using {lowercaseUsingFields[strings.ToLower(field)] = true@@ -672,7 +77,7 @@ func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.Tfor _, field := range l.GetQuerySpanResult() {columns = append(columns, field)if _, ok := lowercaseUsingFields[strings.ToLower(field.Name)]; ok {- delete(rightFieldsMap, field.Name)+ delete(rightFieldsMap, strings.ToLower(field.Name))}}for _, field := range r.GetQuerySpanResult() {@@ -681,33 +86,24 @@ func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.T}}return &base.PseudoTable{- Name: "",Columns: columns,}case NaturalInnerJoin, NaturalLeftOuterJoin, NaturalRightOuterJoin:- // Natural join will merge all the columns with the same name.- leftFieldsMap := make(map[string]bool)- for _, field := range l.GetQuerySpanResult() {- leftFieldsMap[strings.ToLower(field.Name)] = true- }rightFieldsMap := make(map[string]bool)for _, field := range r.GetQuerySpanResult() {rightFieldsMap[strings.ToLower(field.Name)] = true}-var columns []base.QuerySpanResultfor _, field := range l.GetQuerySpanResult() {columns = append(columns, field)delete(rightFieldsMap, strings.ToLower(field.Name))}-for _, field := range r.GetQuerySpanResult() {if _, ok := rightFieldsMap[strings.ToLower(field.Name)]; ok {columns = append(columns, field)}}return &base.PseudoTable{- Name: "",Columns: columns,}default:@@ -715,460 +111,6 @@ func joinTableSources(l, r base.TableSource, tp joinType, using []string) base.T}}-func (q *querySpanExtractor) extractTableFactor(ctx parser.ITableFactorContext) (base.TableSource, error) {- switch {- case ctx.SingleTable() != nil:- return q.extractSingleTable(ctx.SingleTable())- case ctx.SingleTableParens() != nil:- return q.extractSingleTableParens(ctx.SingleTableParens())- case ctx.DerivedTable() != nil:- return q.extractDerivedTable(ctx.DerivedTable())- case ctx.TableReferenceListParens() != nil:- // TableReferenceListParens is tableFactor rules it a MySQL syntax extension,- // for instance:- // SELECT * FORM (t1, t2) JOIN t3 ON 1;- // This syntax is quivalent to- // SELECT * FROM (t1 CROSS JOIN t2) JOIN t3 ON 1;- tableSources, err := q.extractTableReferenceListParens(ctx.TableReferenceListParens())- if err != nil {- return nil, err- }- var result base.TableSource- for i, tableSource := range tableSources {- q.tableSourceFrom = append(q.tableSourceFrom, tableSource)- if i == 0 {- result = tableSource- continue- }- result = joinTableSources(result, tableSource, CrossJoin, nil)- }- return result, nil- case ctx.TableFunction() != nil:- return q.extractTableFunction(ctx.TableFunction())- default:- panic("unreachable")- }-}--func (q *querySpanExtractor) extractTableFunction(ctx parser.ITableFunctionContext) (base.TableSource, error) {- if ctx == nil {- return nil, nil- }-- name, sourceColumnSet, isPlain, err := q.extractSourceColumnSetFromExpr(ctx.Expr())- if err != nil {- return nil, err- }-- columnList := mysqlExtractColumnsClause(ctx.ColumnsClause())-- if ctx.TableAlias() != nil {- name = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())- } else if name == "" {- name = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Expr())- }-- var result []base.QuerySpanResult- for _, column := range columnList {- result = append(result, base.QuerySpanResult{- Name: column,- SourceColumns: sourceColumnSet,- IsPlainField: isPlain,- })- }-- return &base.PseudoTable{- Name: name,- Columns: result,- }, nil-}--func (q *querySpanExtractor) extractTableReferenceListParens(ctx parser.ITableReferenceListParensContext) ([]base.TableSource, error) {- switch {- case ctx.TableReferenceList() != nil:- return q.extractTableReferenceList(ctx.TableReferenceList())- case ctx.TableReferenceListParens() != nil:- return q.extractTableReferenceListParens(ctx.TableReferenceListParens())- default:- panic("unreachable")- }-}--func (q *querySpanExtractor) extractSubquery(ctx parser.ISubqueryContext) (*base.PseudoTable, error) {- return q.extractQueryExpressionParens(ctx.QueryExpressionParens())-}--func (q *querySpanExtractor) extractDerivedTable(ctx parser.IDerivedTableContext) (base.TableSource, error) {- tableSource, err := q.extractSubquery(ctx.Subquery())- if err != nil {- return nil, err- }-- var aliasName string- if ctx.TableAlias() != nil {- aliasName = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())- }- if aliasName != "" {- tableSource = &base.PseudoTable{- Name: aliasName,- Columns: tableSource.GetQuerySpanResult(),- }- }-- if ctx.ColumnInternalRefList() != nil {- columnList := extractColumnInternalRefList(ctx.ColumnInternalRefList())- if len(columnList) != len(tableSource.GetQuerySpanResult()) {- return nil, errors.Errorf("column list length %d doesn't match the derived table column length %d", len(columnList), len(tableSource.GetQuerySpanResult()))- }- for i := range columnList {- tableSource.GetQuerySpanResult()[i].Name = columnList[i]- }- }-- return tableSource, nil-}--func extractColumnInternalRefList(ctx parser.IColumnInternalRefListContext) []string {- if ctx == nil {- return nil- }-- var result []string- for _, columnInternalRef := range ctx.AllColumnInternalRef() {- result = append(result, NormalizeMySQLIdentifier(columnInternalRef.Identifier()))- }- return result-}--func (q *querySpanExtractor) extractSingleTable(ctx parser.ISingleTableContext) (base.TableSource, error) {- databaseName, tableName := NormalizeMySQLTableRef(ctx.TableRef())- tableSource, err := q.findTableSchema(databaseName, tableName)- if err != nil {- return nil, err- }-- var aliasName string- if ctx.TableAlias() != nil {- aliasName = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())- }-- if aliasName != "" {- return &base.PseudoTable{- Name: aliasName,- Columns: tableSource.GetQuerySpanResult(),- }, nil- }- return tableSource, nil-}--func (q *querySpanExtractor) extractSingleTableParens(ctx parser.ISingleTableParensContext) (base.TableSource, error) {- switch {- case ctx.SingleTable() != nil:- return q.extractSingleTable(ctx.SingleTable())- case ctx.SingleTableParens() != nil:- return q.extractSingleTableParens(ctx.SingleTableParens())- default:- panic("unreachable")- }-}--// extractCommonTableExpression extracts the pseudo table from the common table expression.-func (q *querySpanExtractor) extractCommonTableExpression(ctx parser.ICommonTableExpressionContext, recursive bool) (*base.PseudoTable, error) {- if recursive {- return q.extractRecursiveCTE(ctx)- }- return q.extractNonRecursiveCTE(ctx)-}--// extractRecursiveCTE extracts the pseudo table from the recursive common table expression.-func (q *querySpanExtractor) extractRecursiveCTE(ctx parser.ICommonTableExpressionContext) (*base.PseudoTable, error) {- cteName := NormalizeMySQLIdentifier(ctx.Identifier())- l := &recursiveCTEExtractListener{- extractor: q,- cteInfo: &base.PseudoTable{- Name: cteName,- Columns: []base.QuerySpanResult{},- },- selfName: cteName,- foundFirstQueryExpressionBody: false,- inCTE: false,- }- if ctx.ColumnInternalRefList() != nil {- columnList := mysqlExtractColumnInternalRefList(ctx.ColumnInternalRefList())- for i := range columnList {- l.cteInfo.Columns = append(l.cteInfo.Columns, base.QuerySpanResult{- Name: columnList[i],- SourceColumns: make(base.SourceColumnSet),- })- }- }- antlr.ParseTreeWalkerDefault.Walk(l, ctx.Subquery())- if l.err != nil {- return nil, l.err- }-- return l.cteInfo, nil-}--// extractNonRecursiveCTE extracts the pseudo table from the non-recursive common table expression.-func (q *querySpanExtractor) extractNonRecursiveCTE(ctx parser.ICommonTableExpressionContext) (*base.PseudoTable, error) {- tableSource, err := q.extractSubquery(ctx.Subquery())- if err != nil {- return nil, err- }- spanResults := tableSource.GetQuerySpanResult()- if ctx.ColumnInternalRefList() != nil {- columnList := mysqlExtractColumnInternalRefList(ctx.ColumnInternalRefList())- if len(columnList) != len(tableSource.GetQuerySpanResult()) {- return nil, errors.Errorf("MySQL CTE column list should have the same length, but got %d and %d", len(columnList), len(tableSource.GetQuerySpanResult()))- }- for i := range columnList {- spanResults[i].Name = columnList[i]- }- }- cteName := NormalizeMySQLIdentifier(ctx.Identifier())- result := &base.PseudoTable{- Name: cteName,- Columns: spanResults,- }- return result, nil-}--type recursiveCTEExtractListener struct {- *parser.BaseMySQLParserListener-- extractor *querySpanExtractor- cteInfo *base.PseudoTable- selfName string- outerCTEs []parser.IWithClauseContext- foundFirstQueryExpressionBody bool- inCTE bool- err error-}--// EnterQueryExpression is called when production queryExpression is entered.-func (l *recursiveCTEExtractListener) EnterQueryExpression(ctx *parser.QueryExpressionContext) {- if l.foundFirstQueryExpressionBody || l.inCTE || l.err != nil {- return- }- if ctx.WithClause() != nil {- l.outerCTEs = append(l.outerCTEs, ctx.WithClause())- }-}--// EnterWithClause is called when production commonTableExpression is entered.-func (l *recursiveCTEExtractListener) EnterWithClause(_ *parser.WithClauseContext) {- l.inCTE = true-}--// ExitWithClause is called when production commonTableExpression is exited.-func (l *recursiveCTEExtractListener) ExitWithClause(_ *parser.WithClauseContext) {- l.inCTE = false-}--// EnterQueryExpressionBody is called when production queryExpressionBody is entered.-func (l *recursiveCTEExtractListener) EnterQueryExpressionBody(ctx *parser.QueryExpressionBodyContext) {- if l.err != nil {- return- }- if l.inCTE {- return- }- if l.foundFirstQueryExpressionBody {- return- }-- l.foundFirstQueryExpressionBody = true-- // Deal with outer CTEs.- cetOuterLength := len(l.extractor.ctes)- defer func() {- l.extractor.ctes = l.extractor.ctes[:cetOuterLength]- }()- for _, outerCTE := range l.outerCTEs {- recursive := outerCTE.RECURSIVE_SYMBOL() != nil- for _, cte := range outerCTE.AllCommonTableExpression() {- cteTable, err := l.extractor.extractCommonTableExpression(cte, recursive)- if err != nil {- l.err = err- return- }- l.extractor.ctes = append(l.extractor.ctes, cteTable)- }- }-- var initialPart []base.QuerySpanResult- var recursivePart []antlr.ParserRuleContext-- findRecursivePart := false- for _, child := range ctx.GetChildren() {- switch child := child.(type) {- case *parser.QueryPrimaryContext:- if !findRecursivePart {- resource := extractTableRefs("", child)-- for _, item := range resource {- if item.Database == "" && item.Table == l.selfName {- findRecursivePart = true- break- }- }- }-- if findRecursivePart {- recursivePart = append(recursivePart, child)- } else {- tableSource, err := l.extractor.extractQueryPrimary(child)- if err != nil {- l.err = err- return- }- if len(initialPart) == 0 {- initialPart = tableSource.GetQuerySpanResult()- } else {- if len(initialPart) != len(tableSource.GetQuerySpanResult()) {- l.err = errors.Errorf("MySQL UNION field list should have the same length, but got %d and %d", len(initialPart), len(tableSource.GetQuerySpanResult()))- return- }- for i := range initialPart {- initialPart[i].SourceColumns, _ = base.MergeSourceColumnSet(initialPart[i].SourceColumns, tableSource.GetQuerySpanResult()[i].SourceColumns)- }- }- }- case *parser.QueryExpressionParensContext:- queryExpression := extractQueryExpression(child)- if queryExpression == nil {- // Never happen.- l.err = errors.Errorf("MySQL query expression parens should have query expression, but got nil")- return- }-- if !findRecursivePart {- resource := extractTableRefs("", queryExpression)-- for _, item := range resource {- if item.Database == "" && item.Table == l.selfName {- findRecursivePart = true- break- }- }- }-- if findRecursivePart {- recursivePart = append(recursivePart, child)- } else {- tableSource, err := l.extractor.extractQueryExpression(queryExpression)- if err != nil {- l.err = err- return- }- if len(initialPart) == 0 {- initialPart = tableSource.GetQuerySpanResult()- } else {- if len(initialPart) != len(tableSource.GetQuerySpanResult()) {- l.err = errors.Errorf("MySQL UNION field list should have the same length, but got %d and %d", len(initialPart), len(tableSource.GetQuerySpanResult()))- return- }- for i := range initialPart {- initialPart[i].SourceColumns, _ = base.MergeSourceColumnSet(initialPart[i].SourceColumns, tableSource.GetQuerySpanResult()[i].SourceColumns)- }- }- }- default:- }- }-- // Compute dependent closures.- // There are two ways to compute dependent closures:- // 1. find the all dependent edges, then use graph theory traversal to find the closure.- // 2. Iterate to simulate the CTE recursive process, each turn check whether the Sensitive state has changed, and stop if no change.- //- // Consider the option 2 can easy to implementation, because the simulate process has been written.- // On the other hand, the number of iterations of the entire algorithm will not exceed the length of fields.- // In actual use, the length of fields will not be more than 20 generally.- // So I think it's OK for now.- // If any performance issues in use, optimize here.- if len(l.cteInfo.GetQuerySpanResult()) == 0 {- for _, item := range initialPart {- l.cteInfo.Columns = append(l.cteInfo.Columns, base.QuerySpanResult{- Name: item.Name,- SourceColumns: item.SourceColumns,- })- }- } else {- if len(initialPart) != len(l.cteInfo.GetQuerySpanResult()) {- l.err = errors.Errorf("The common table expression and column names list have different column counts")- return- }- for i := range initialPart {- l.cteInfo.Columns[i].SourceColumns, _ = base.MergeSourceColumnSet(initialPart[i].SourceColumns, l.cteInfo.GetQuerySpanResult()[i].SourceColumns)- }- }-- if len(recursivePart) == 0 {- return- }-- l.extractor.ctes = append(l.extractor.ctes, l.cteInfo)- defer func() {- l.extractor.ctes = l.extractor.ctes[:len(l.extractor.ctes)-1]- }()- for {- var fieldList []base.QuerySpanResult- for _, item := range recursivePart {- var itemFields []base.QuerySpanResult- switch item := item.(type) {- case *parser.QueryPrimaryContext:- var err error- tableSource, err := l.extractor.extractQueryPrimary(item)- if err != nil {- l.err = err- return- }- itemFields = tableSource.GetQuerySpanResult()- case *parser.QueryExpressionContext:- var err error- tableSource, err := l.extractor.extractQueryExpression(item)- if err != nil {- l.err = err- return- }- itemFields = tableSource.GetQuerySpanResult()- default:- }- if len(fieldList) == 0 {- fieldList = itemFields- } else {- if len(fieldList) != len(itemFields) {- l.err = errors.Errorf("MySQL UNION field list should have the same length, but got %d and %d", len(fieldList), len(itemFields))- return- }- for i := range fieldList {- fieldList[i].SourceColumns, _ = base.MergeSourceColumnSet(fieldList[i].SourceColumns, itemFields[i].SourceColumns)- }- }- }-- if len(fieldList) != len(l.cteInfo.GetQuerySpanResult()) {- // The error content comes from MySQL.- l.err = errors.Errorf("The common table expression and column names list have different column counts")- return- }-- changed := false- for i, field := range fieldList {- var ok bool- l.cteInfo.Columns[i].SourceColumns, ok = base.MergeSourceColumnSet(l.cteInfo.GetQuerySpanResult()[i].SourceColumns, field.SourceColumns)- changed = changed || ok- }-- if !changed {- break- }- l.extractor.ctes[len(l.extractor.ctes)-1] = l.cteInfo- }-}-func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName string) ([]base.QuerySpanResult, bool) {findInTableSource := func(tableSource base.TableSource) ([]base.QuerySpanResult, bool) {if q.ignoreCaseSensitive {@@ -1186,37 +128,19 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName streturn nil, false}}- // If the table name is empty, we should check if there are ambiguous fields,- // but we delegate this responsibility to the db-server, we do the fail-open strategy here.-return tableSource.GetQuerySpanResult(), true}- // One sub-query may have multi-outer schemas and the multi-outer schemas can use the same name, such as:- //- // select (- // select (- // select max(a) > x1.a from t- // )- // from t1 as x1- // limit 1- // )- // from t as x1;- //- // This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.- // This is the reason we loop the slice in reversed order.for i := len(q.outerTableSources) - 1; i >= 0; i-- {if querySpanResult, ok := findInTableSource(q.outerTableSources[i]); ok {return querySpanResult, true}}-for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {if querySpanResult, ok := findInTableSource(q.tableSourceFrom[i]); ok {return querySpanResult, true}}-return nil, false}@@ -1238,11 +162,7 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldreturn nil, false}}- // If the table name is empty, we should check if there are ambiguous fields,- // but we delegate this responsibility to the db-server, we do the fail-open strategy here.-- querySpanResult := tableSource.GetQuerySpanResult()- for _, column := range querySpanResult {+ for _, column := range tableSource.GetQuerySpanResult() {if strings.EqualFold(column.Name, fieldName) {return column.SourceColumns, true}@@ -1250,31 +170,16 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldreturn nil, false}- // One sub-query may have multi-outer schemas and the multi-outer schemas can use the same name, such as:- //- // select (- // select (- // select max(a) > x1.a from t- // )- // from t1 as x1- // limit 1- // )- // from t as x1;- //- // This query has two tables can be called `x1`, and the expression x1.a uses the closer x1 table.- // This is the reason we loop the slice in reversed order.for i := len(q.outerTableSources) - 1; i >= 0; i-- {if sourceColumnSet, ok := findInTableSource(q.outerTableSources[i]); ok {return sourceColumnSet, nil}}-for i := len(q.tableSourceFrom) - 1; i >= 0; i-- {if sourceColumnSet, ok := findInTableSource(q.tableSourceFrom[i]); ok {return sourceColumnSet, nil}}-for i := len(q.priorTableInFrom) - 1; i >= 0; i-- {if sourceColumnSet, ok := findInTableSource(q.priorTableInFrom[i]); ok {return sourceColumnSet, nil@@ -1290,8 +195,6 @@ func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldfunc (q *querySpanExtractor) filterClusterName(databaseName string) string {if q.gCtx.Engine == storepb.Engine_STARROCKS {- // For StarRocks, user can use `cluster_name:database_name` as the database name.- // But for the query span in Bytebase, we only care about the database name.list := strings.Split(databaseName, ":")if len(list) > 1 {databaseName = list[len(list)-1]@@ -1301,15 +204,6 @@ func (q *querySpanExtractor) filterClusterName(databaseName string) string {}func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (base.TableSource, error) {- // Each CTE name in one WITH clause must be unique, but we can use the same name in the different level CTE, such as:- //- // with tt2 as (- // with tt2 as (select * from t)- // select max(a) from tt2)- // select * from tt2- //- // This query has two CTE can be called `tt2`, and the FROM clause 'from tt2' uses the closer tt2 CTE.- // This is the reason we loop the slice in reversed order.if databaseName == "" {for i := len(q.ctes) - 1; i >= 0; i-- {table := q.ctes[i]@@ -1322,7 +216,6 @@ func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (baif databaseName == "" {databaseName = q.defaultDatabase}-databaseName = q.filterClusterName(databaseName)var dbMetadata *model.DatabaseMetadata@@ -1454,88 +347,6 @@ func cloneViewResolutionStack(in map[string]bool) map[string]bool {return out}-// selectOnlyListener is the listener to listen the top level select statement only.-type selectOnlyListener struct {- *parser.BaseMySQLParserListener-- // The only misson of the listener is the find the precise select statement.- // All the eval work will be handled by the querySpanExtractor.- extractor *querySpanExtractor- querySpan *base.QuerySpan- err error-}--func newSelectOnlyListener(extractor *querySpanExtractor) *selectOnlyListener {- return &selectOnlyListener{- extractor: extractor,- querySpan: &base.QuerySpan{- Results: []base.QuerySpanResult{},- SourceColumns: make(base.SourceColumnSet),- },- }-}--func (s *selectOnlyListener) EnterSelectStatement(ctx *parser.SelectStatementContext) {- parent := ctx.GetParent()- if parent == nil {- return- }-- if _, ok := parent.(*parser.SimpleStatementContext); !ok {- return- }-- fields, err := s.extractor.extractContext(ctx)- if err != nil {- s.err = err- return- }-- s.querySpan.Results = append(s.querySpan.Results, fields.Columns...)-}--func getAccessTables(currentDatabase string, tree antlr.Tree) base.SourceColumnSet {- l := newAccessTableListener(currentDatabase)-- result := make(base.SourceColumnSet)- antlr.ParseTreeWalkerDefault.Walk(l, tree)- result, _ = base.MergeSourceColumnSet(result, l.sourceColumnSet)-- return result-}--type accessTableListener struct {- *parser.BaseMySQLParserListener-- currentDatabase string- sourceColumnSet base.SourceColumnSet-}--func newAccessTableListener(currentDatabase string) *accessTableListener {- return &accessTableListener{- currentDatabase: currentDatabase,- sourceColumnSet: make(base.SourceColumnSet),- }-}--// EnterTableRef is called when production tableRef is entered.-func (l *accessTableListener) EnterTableRef(ctx *parser.TableRefContext) {- sourceColumn := base.ColumnResource{- Database: l.currentDatabase,- }- if ctx.DotIdentifier() != nil {- sourceColumn.Table = NormalizeMySQLIdentifier(ctx.DotIdentifier().Identifier())- }- db, table := normalizeMySQLQualifiedIdentifier(ctx.QualifiedIdentifier())- if db != "" {- sourceColumn.Database = db- }- sourceColumn.Table = table- l.sourceColumnSet[sourceColumn] = true-}--// isMixedQuery checks whether the query accesses the user table and system table at the same time.-// It returns whether all tables are system tables and whether there is a mixture.func isMixedQuery(m base.SourceColumnSet, ignoreCaseSensitive bool) (bool, bool) {hasSystem, hasUser := false, falsefor table := range m {@@ -1545,26 +356,17 @@ func isMixedQuery(m base.SourceColumnSet, ignoreCaseSensitive bool) (bool, bool)hasUser = true}}-if hasSystem && hasUser {return false, true}-return !hasUser && hasSystem, false}-// reservedSystemDatabases are schema names MySQL itself reserves regardless of-// lower_case_table_names. A user cannot create a schema whose lowercase name-// collides with any of these, so matching is always case-insensitive.var reservedSystemDatabases = map[string]bool{"information_schema": true,"performance_schema": true,}-// onDiskSystemDatabases are system schemas stored on disk. On case-sensitive-// instances (lower_case_table_names=0), users can legitimately create a distinct-// schema whose name differs from these only by case, so we only match them-// case-insensitively when the instance is case-insensitive.var onDiskSystemDatabases = map[string]bool{"mysql": true,}@@ -1579,104 +381,3 @@ func isSystemResource(resource base.ColumnResource, ignoreCaseSensitive bool) bo}return onDiskSystemDatabases[database]}--func mysqlExtractColumnsClause(ctx parser.IColumnsClauseContext) []string {- if ctx == nil {- return nil- }-- var result []string- for _, column := range ctx.AllJtColumn() {- result = append(result, mysqlExtractJtColumn(column)...)- }-- return result-}--func mysqlExtractColumnInternalRefList(ctx parser.IColumnInternalRefListContext) []string {- if ctx == nil {- return nil- }-- var result []string- for _, columnInternalRef := range ctx.AllColumnInternalRef() {- result = append(result, NormalizeMySQLIdentifier(columnInternalRef.Identifier()))- }- return result-}--func extractQueryExpression(ctx parser.IQueryExpressionParensContext) parser.IQueryExpressionContext {- if ctx == nil {- return nil- }-- switch {- case ctx.QueryExpression() != nil:- return ctx.QueryExpression()- case ctx.QueryExpressionParens() != nil:- return extractQueryExpression(ctx.QueryExpressionParens())- default:- return nil- }-}--func mysqlExtractJtColumn(ctx parser.IJtColumnContext) []string {- if ctx == nil {- return []string{}- }-- switch {- case ctx.Identifier() != nil:- return []string{NormalizeMySQLIdentifier(ctx.Identifier())}- case ctx.ColumnsClause() != nil:- return mysqlExtractColumnsClause(ctx.ColumnsClause())- default:- return []string{}- }-}--func extractTableRefs(database string, ctx antlr.ParserRuleContext) []base.SchemaResource {- l := &resourceExtractListener{- currentDatabase: database,- resourceMap: make(map[string]base.SchemaResource),- }-- var result []base.SchemaResource- antlr.ParseTreeWalkerDefault.Walk(l, ctx)- for _, resource := range l.resourceMap {- result = append(result, resource)- }-- slices.SortFunc(result, func(i, j base.SchemaResource) int {- if i.String() < j.String() {- return -1- }- if i.String() > j.String() {- return 1- }- return 0- })-- return result-}--type resourceExtractListener struct {- *parser.BaseMySQLParserListener-- currentDatabase string- resourceMap map[string]base.SchemaResource-}--// EnterTableRef is called when production tableRef is entered.-func (l *resourceExtractListener) EnterTableRef(ctx *parser.TableRefContext) {- resource := base.SchemaResource{Database: l.currentDatabase}- if ctx.DotIdentifier() != nil {- resource.Table = NormalizeMySQLIdentifier(ctx.DotIdentifier().Identifier())- }- db, table := normalizeMySQLQualifiedIdentifier(ctx.QualifiedIdentifier())- if db != "" {- resource.Database = db- }- resource.Table = table- l.resourceMap[resource.String()] = resource-}diff --git a/backend/plugin/parser/mysql/query_span_extractor_omni.go b/backend/plugin/parser/mysql/query_span_extractor_omni.gonew file mode 100644index 0000000000..b455c437b5--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_extractor_omni.go@@ -0,0 +1,1429 @@+package mysql++import (+ "context"+ "strconv"+ "strings"++ "github.com/bytebase/omni/mysql/ast"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// omniQuerySpanExtractor extracts query span information from omni MySQL AST.+// It embeds querySpanExtractor so shared metadata and identifier-resolution+// helpers can be reused as the omni implementation grows.+type omniQuerySpanExtractor struct {+ *querySpanExtractor+ source string+}++func newOmniQuerySpanExtractor(defaultDatabase string, gCtx base.GetQuerySpanContext, ignoreCaseSensitive bool) *omniQuerySpanExtractor {+ return &omniQuerySpanExtractor{+ querySpanExtractor: newQuerySpanExtractor(defaultDatabase, gCtx, ignoreCaseSensitive),+ }+}++// getOmniQuerySpan is the package-internal omni entry point used by migration+// tests and the production MySQL query-span path after cutover.+func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statement string) (*base.QuerySpan, error) {+ q.ctx = ctx+ q.source = statement++ stmts, err := ParseMySQLOmni(statement)+ if err != nil {+ return nil, err+ }+ if stmts == nil || len(stmts.Items) == 0 {+ return &base.QuerySpan{+ Type: base.Select,+ SourceColumns: base.SourceColumnSet{},+ Results: []base.QuerySpanResult{},+ PredicateColumns: base.SourceColumnSet{},+ }, nil+ }+ if len(stmts.Items) != 1 {+ return nil, errors.Errorf("expected exactly 1 statement, got %d", len(stmts.Items))+ }++ root := stmts.Items[0]+ accessTables := collectOmniAccessTables(root, q.defaultDatabase, q.gCtx.Engine == storepb.Engine_STARROCKS)+ allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)+ if mixed {+ return nil, base.MixUserSystemTablesError+ }++ queryType := classifyOmniQueryType(root, allSystems, q.source)+ if allSystems {+ accessTables = base.SourceColumnSet{}+ }+ resultSourceColumns := accessTables+ if queryType != base.Select {+ resultSourceColumns = base.SourceColumnSet{}+ }++ var results []base.QuerySpanResult+ if queryType == base.Select {+ tableSource, err := q.extractOmniSelectRoot(root)+ if err != nil {+ var resourceNotFound *base.ResourceNotFoundError+ if errors.As(err, &resourceNotFound) {+ return &base.QuerySpan{+ Type: base.Select,+ SourceColumns: accessTables,+ Results: []base.QuerySpanResult{},+ PredicateColumns: base.SourceColumnSet{},+ NotFoundError: resourceNotFound,+ }, nil+ }+ return nil, err+ }+ results = tableSource.GetQuerySpanResult()+ }+ return &base.QuerySpan{+ Type: queryType,+ SourceColumns: resultSourceColumns,+ Results: results,+ PredicateColumns: base.SourceColumnSet{},+ }, nil+}++func (q *omniQuerySpanExtractor) extractOmniSelectRoot(root ast.Node) (*base.PseudoTable, error) {+ switch n := root.(type) {+ case *ast.SelectStmt:+ return q.extractOmniSelectStmt(n)+ case *ast.ExplainStmt:+ return &base.PseudoTable{}, nil+ case *ast.TableStmt:+ return q.extractOmniTableStmt(n)+ case *ast.ValuesStmt:+ return q.extractOmniValuesStmt(n)+ default:+ return &base.PseudoTable{}, nil+ }+}++func (q *omniQuerySpanExtractor) extractOmniSelectStmt(stmt *ast.SelectStmt) (*base.PseudoTable, error) {+ if stmt == nil {+ return &base.PseudoTable{}, nil+ }+ if stmt.Into != nil {+ return nil, errors.New("meet unsupported select statement with into")+ }++ originalTableSourceFrom := q.tableSourceFrom+ originalPriorLength := len(q.priorTableInFrom)+ originalCTELength := len(q.ctes)+ defer func() {+ q.tableSourceFrom = originalTableSourceFrom+ q.priorTableInFrom = q.priorTableInFrom[:originalPriorLength]+ q.ctes = q.ctes[:originalCTELength]+ }()++ if err := q.processOmniCTEs(stmt.CTEs); err != nil {+ return nil, err+ }+ if stmt.SetOp != ast.SetOpNone {+ return q.extractOmniSetOp(stmt)+ }++ fromSources, err := q.extractOmniTableSources(stmt.From)+ if err != nil {+ return nil, err+ }+ q.tableSourceFrom = append(q.tableSourceFrom, fromSources...)++ results, err := q.extractOmniTargetList(stmt.TargetList, fromSources)+ if err != nil {+ return nil, err+ }+ return &base.PseudoTable{+ Columns: results,+ }, nil+}++func (q *omniQuerySpanExtractor) extractOmniTableStmt(stmt *ast.TableStmt) (*base.PseudoTable, error) {+ if stmt == nil || stmt.Table == nil {+ return &base.PseudoTable{}, nil+ }+ if stmt.Into != nil {+ return nil, errors.New("meet unsupported table statement with into")+ }+ tableSource, err := q.findTableSchema(stmt.Table.Schema, stmt.Table.Name)+ if err != nil {+ return nil, err+ }+ return &base.PseudoTable{Columns: tableSource.GetQuerySpanResult()}, nil+}++func (q *omniQuerySpanExtractor) extractOmniValuesStmt(stmt *ast.ValuesStmt) (*base.PseudoTable, error) {+ if stmt == nil || len(stmt.Rows) == 0 {+ return &base.PseudoTable{}, nil+ }+ var columns []base.QuerySpanResult+ for _, expr := range stmt.Rows[0] {+ field, err := q.extractOmniExpr(expr)+ if err != nil {+ return nil, err+ }+ columns = append(columns, field)+ }+ return &base.PseudoTable{Columns: columns}, nil+}++func (q *omniQuerySpanExtractor) extractOmniSetOp(stmt *ast.SelectStmt) (*base.PseudoTable, error) {+ if stmt == nil || stmt.SetOp == ast.SetOpNone {+ return q.extractOmniSelectStmt(stmt)+ }+ left, err := q.extractOmniSelectStmt(stmt.Left)+ if err != nil {+ return nil, err+ }+ right, err := q.extractOmniSelectStmt(stmt.Right)+ if err != nil {+ return nil, err+ }+ leftResults := left.GetQuerySpanResult()+ rightResults := right.GetQuerySpanResult()+ if len(leftResults) != len(rightResults) {+ return nil, errors.Errorf("MySQL UNION operator left has %d fields, right has %d fields", len(leftResults), len(rightResults))+ }+ results := make([]base.QuerySpanResult, 0, len(leftResults))+ for i := range leftResults {+ sourceColumns, _ := base.MergeSourceColumnSet(leftResults[i].SourceColumns, rightResults[i].SourceColumns)+ results = append(results, base.QuerySpanResult{+ Name: leftResults[i].Name,+ SourceColumns: sourceColumns,+ IsPlainField: false,+ })+ }+ return &base.PseudoTable{Columns: results}, nil+}++func (q *omniQuerySpanExtractor) processOmniCTEs(ctes []*ast.CommonTableExpr) error {+ for _, cte := range ctes {+ tableSource, err := q.extractOmniCTE(cte)+ if err != nil {+ return err+ }+ if tableSource != nil {+ q.ctes = append(q.ctes, tableSource)+ }+ }+ return nil+}++func (q *omniQuerySpanExtractor) extractOmniCTE(cte *ast.CommonTableExpr) (*base.PseudoTable, error) {+ if cte == nil {+ return nil, nil+ }+ if cte.Recursive {+ return q.extractOmniRecursiveCTE(cte)+ }+ return q.extractOmniNonRecursiveCTE(cte)+}++func (q *omniQuerySpanExtractor) extractOmniNonRecursiveCTE(cte *ast.CommonTableExpr) (*base.PseudoTable, error) {+ tableSource, err := q.extractOmniSelectStmt(cte.Select)+ if err != nil {+ return nil, err+ }+ results := cloneOmniQuerySpanResults(tableSource.GetQuerySpanResult())+ if len(cte.Columns) > 0 {+ if len(cte.Columns) != len(results) {+ return nil, errors.Errorf("MySQL CTE column list should have the same length, but got %d and %d", len(cte.Columns), len(results))+ }+ for i := range cte.Columns {+ results[i].Name = cte.Columns[i]+ }+ }+ return &base.PseudoTable{Name: cte.Name, Columns: results}, nil+}++func (q *omniQuerySpanExtractor) extractOmniRecursiveCTE(cte *ast.CommonTableExpr) (*base.PseudoTable, error) {+ if cte.Select == nil || cte.Select.SetOp == ast.SetOpNone {+ return q.extractOmniNonRecursiveCTE(cte)+ }+ initialTable, err := q.extractOmniRecursiveCTEAnchor(cte.Select)+ if err != nil {+ return nil, err+ }+ results := cloneOmniQuerySpanResults(initialTable.GetQuerySpanResult())+ if len(cte.Columns) > 0 {+ if len(cte.Columns) != len(results) {+ return nil, errors.Errorf("The common table expression and column names list have different column counts")+ }+ for i := range cte.Columns {+ results[i].Name = cte.Columns[i]+ }+ }+ cteInfo := &base.PseudoTable{Name: cte.Name, Columns: results}+ q.ctes = append(q.ctes, cteInfo)+ defer func() {+ q.ctes = q.ctes[:len(q.ctes)-1]+ }()++ for {+ recursiveTable, err := q.extractOmniSelectStmt(cte.Select)+ if err != nil {+ return nil, err+ }+ recursiveResults := recursiveTable.GetQuerySpanResult()+ if len(recursiveResults) != len(cteInfo.Columns) {+ return nil, errors.Errorf("The common table expression and column names list have different column counts")+ }+ changed := false+ for i := range recursiveResults {+ var ok bool+ cteInfo.Columns[i].SourceColumns, ok = base.MergeSourceColumnSet(cteInfo.Columns[i].SourceColumns, recursiveResults[i].SourceColumns)+ cteInfo.Columns[i].IsPlainField = false+ changed = changed || ok+ }+ if !changed {+ break+ }+ }+ return cteInfo, nil+}++func (q *omniQuerySpanExtractor) extractOmniRecursiveCTEAnchor(stmt *ast.SelectStmt) (*base.PseudoTable, error) {+ if stmt == nil || stmt.SetOp == ast.SetOpNone {+ return q.extractOmniSelectStmt(stmt)+ }+ if stmt.Left != nil {+ return q.extractOmniRecursiveCTEAnchor(stmt.Left)+ }+ return q.extractOmniRecursiveCTEAnchor(stmt.Right)+}++func cloneOmniQuerySpanResults(results []base.QuerySpanResult) []base.QuerySpanResult {+ clone := make([]base.QuerySpanResult, len(results))+ copy(clone, results)+ return clone+}++func (q *omniQuerySpanExtractor) extractOmniTableSources(tableExprs []ast.TableExpr) ([]base.TableSource, error) {+ var result []base.TableSource+ for _, tableExpr := range tableExprs {+ tableSource, err := q.extractOmniTableSource(tableExpr)+ if err != nil {+ return nil, err+ }+ if tableSource == nil {+ continue+ }+ q.priorTableInFrom = append(q.priorTableInFrom, tableSource)+ result = append(result, tableSource)+ }+ return result, nil+}++func (q *omniQuerySpanExtractor) extractOmniTableSource(tableExpr ast.TableExpr) (base.TableSource, error) {+ switch t := tableExpr.(type) {+ case nil:+ return nil, nil+ case *ast.TableRef:+ if isOmniDualTable(t) {+ return &base.PseudoTable{Name: t.Name}, nil+ }+ tableSource, err := q.findTableSchema(t.Schema, t.Name)+ if err != nil {+ return nil, err+ }+ if t.Alias == "" {+ return tableSource, nil+ }+ return &base.PseudoTable{+ Name: t.Alias,+ Columns: tableSource.GetQuerySpanResult(),+ }, nil+ case *ast.SubqueryExpr:+ return q.extractOmniSubquery(t)+ case *ast.JoinClause:+ return q.extractOmniJoin(t)+ case *ast.JsonTableExpr:+ return q.extractOmniJSONTable(t)+ default:+ return nil, errors.Errorf("unsupported omni MySQL table source %T", tableExpr)+ }+}++func (q *omniQuerySpanExtractor) extractOmniJSONTable(expr *ast.JsonTableExpr) (base.TableSource, error) {+ if expr == nil {+ return nil, nil+ }+ name := expr.Alias+ if name == "" {+ name = q.omniExprName(expr.Expr)+ }+ field, err := q.extractOmniExpr(expr.Expr)+ if err != nil {+ return nil, err+ }+ var columns []base.QuerySpanResult+ for _, column := range flattenOmniJSONTableColumns(expr.Columns) {+ columns = append(columns, base.QuerySpanResult{+ Name: column.Name,+ SourceColumns: field.SourceColumns,+ IsPlainField: field.IsPlainField,+ })+ }+ return &base.PseudoTable{Name: name, Columns: columns}, nil+}++func flattenOmniJSONTableColumns(columns []*ast.JsonTableColumn) []*ast.JsonTableColumn {+ var result []*ast.JsonTableColumn+ for _, column := range columns {+ if column == nil {+ continue+ }+ if column.Nested {+ result = append(result, flattenOmniJSONTableColumns(column.NestedCols)...)+ continue+ }+ result = append(result, column)+ }+ return result+}++func (q *omniQuerySpanExtractor) extractOmniJoin(join *ast.JoinClause) (base.TableSource, error) {+ if join == nil {+ return nil, nil+ }+ left, err := q.extractOmniTableSource(join.Left)+ if err != nil {+ return nil, err+ }+ if left != nil {+ q.tableSourceFrom = append(q.tableSourceFrom, left)+ }+ right, err := q.extractOmniTableSource(join.Right)+ if err != nil {+ return nil, err+ }+ if right != nil {+ q.tableSourceFrom = append(q.tableSourceFrom, right)+ }++ var using []string+ if condition, ok := join.Condition.(*ast.UsingCondition); ok && condition != nil {+ using = condition.Columns+ }+ return joinTableSources(left, right, omniJoinType(join.Type), using), nil+}++func omniJoinType(joinType ast.JoinType) joinType {+ switch joinType {+ case ast.JoinInner:+ return InnerJoin+ case ast.JoinLeft:+ return LeftOuterJoin+ case ast.JoinRight:+ return RightOuterJoin+ case ast.JoinCross:+ return CrossJoin+ case ast.JoinNatural:+ return NaturalInnerJoin+ case ast.JoinStraight:+ return StraightJoin+ case ast.JoinNaturalLeft:+ return NaturalLeftOuterJoin+ case ast.JoinNaturalRight:+ return NaturalRightOuterJoin+ default:+ return Join+ }+}++func (q *omniQuerySpanExtractor) extractOmniSubquery(expr *ast.SubqueryExpr) (*base.PseudoTable, error) {+ if expr == nil {+ return &base.PseudoTable{}, nil+ }+ subqueryExtractor := q.cloneOmniForSubquery()+ if expr.Lateral {+ subqueryExtractor.outerTableSources = append(subqueryExtractor.outerTableSources, q.priorTableInFrom...)+ }+ tableSource, err := subqueryExtractor.extractOmniSelectStmt(expr.Select)+ if err != nil {+ return nil, err+ }+ results := cloneOmniQuerySpanResults(tableSource.GetQuerySpanResult())+ if len(expr.Columns) > 0 {+ if len(expr.Columns) != len(results) {+ return nil, errors.Errorf("derived table column list length %d doesn't match result column length %d", len(expr.Columns), len(results))+ }+ for i := range expr.Columns {+ results[i].Name = expr.Columns[i]+ }+ }+ if expr.Alias == "" {+ return &base.PseudoTable{Name: tableSource.Name, Columns: results}, nil+ }+ return &base.PseudoTable{+ Name: expr.Alias,+ Columns: results,+ }, nil+}++func (q *omniQuerySpanExtractor) cloneOmniForSubquery() *omniQuerySpanExtractor {+ return &omniQuerySpanExtractor{+ querySpanExtractor: &querySpanExtractor{+ ctx: q.ctx,+ defaultDatabase: q.defaultDatabase,+ gCtx: q.gCtx,+ ctes: q.ctes,+ outerTableSources: append(q.outerTableSources, q.tableSourceFrom...),+ tableSourceFrom: []base.TableSource{},+ ignoreCaseSensitive: q.ignoreCaseSensitive,+ viewResolutionStack: cloneViewResolutionStack(q.viewResolutionStack),+ },+ source: q.source,+ }+}++func (q *omniQuerySpanExtractor) extractOmniTargetList(targets []ast.ExprNode, fromSources []base.TableSource) ([]base.QuerySpanResult, error) {+ var result []base.QuerySpanResult+ for _, target := range targets {+ switch t := target.(type) {+ case *ast.ResTarget:+ if _, ok := t.Val.(*ast.StarExpr); ok {+ fields, err := q.extractOmniTableWild("", "", fromSources)+ if err != nil {+ return nil, err+ }+ result = append(result, fields...)+ continue+ }+ if columnRef, ok := t.Val.(*ast.ColumnRef); ok && columnRef.Star {+ fields, err := q.extractOmniColumnWild(columnRef, fromSources)+ if err != nil {+ return nil, err+ }+ result = append(result, fields...)+ continue+ }+ case *ast.StarExpr:+ fields, err := q.extractOmniTableWild("", "", fromSources)+ if err != nil {+ return nil, err+ }+ result = append(result, fields...)+ continue+ case *ast.ColumnRef:+ if t.Star {+ fields, err := q.extractOmniColumnWild(t, fromSources)+ if err != nil {+ return nil, err+ }+ result = append(result, fields...)+ continue+ }+ default:+ }++ field, err := q.extractOmniTarget(target)+ if err != nil {+ return nil, err+ }+ result = append(result, field)+ }+ return result, nil+}++func (q *omniQuerySpanExtractor) extractOmniTarget(target ast.ExprNode) (base.QuerySpanResult, error) {+ switch t := target.(type) {+ case *ast.ResTarget:+ field, err := q.extractOmniExpr(t.Val)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ if t.Name != "" {+ field.Name = t.Name+ }+ return field, nil+ default:+ return q.extractOmniExpr(target)+ }+}++func (q *omniQuerySpanExtractor) extractOmniExpr(expr ast.ExprNode) (base.QuerySpanResult, error) {+ switch e := expr.(type) {+ case nil:+ return base.QuerySpanResult{+ SourceColumns: base.SourceColumnSet{},+ IsPlainField: true,+ }, nil+ case *ast.ColumnRef:+ if e.Star {+ return base.QuerySpanResult{}, errors.Errorf("asterisk is not a scalar expression")+ }+ sourceColumns, err := q.getFieldColumnSource(e.Schema, e.Table, e.Column)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{+ Name: e.Column,+ SourceColumns: sourceColumns,+ IsPlainField: true,+ }, nil+ case *ast.StarExpr:+ return base.QuerySpanResult{}, errors.Errorf("asterisk is not a scalar expression")+ case *ast.IntLit, *ast.FloatLit, *ast.StringLit, *ast.BoolLit, *ast.NullLit, *ast.HexLit, *ast.BitLit, *ast.TemporalLit:+ return base.QuerySpanResult{+ Name: q.omniExprName(e),+ SourceColumns: base.SourceColumnSet{},+ IsPlainField: true,+ }, nil+ case *ast.VariableRef:+ return base.QuerySpanResult{+ Name: q.omniExprName(e),+ SourceColumns: base.SourceColumnSet{},+ IsPlainField: true,+ }, nil+ case *ast.ParenExpr:+ field, err := q.extractOmniExpr(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ field.Name = q.omniExprName(e)+ field.IsPlainField = false+ return field, nil+ case *ast.BinaryExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Left, e.Right)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.UnaryExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Operand)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.FuncCallExpr:+ exprs := append([]ast.ExprNode{}, e.Args...)+ exprs = append(exprs, e.Separator)+ for _, item := range e.OrderBy {+ if item != nil {+ exprs = append(exprs, item.Expr)+ }+ }+ if e.Over != nil {+ exprs = append(exprs, e.Over.PartitionBy...)+ for _, item := range e.Over.OrderBy {+ if item != nil {+ exprs = append(exprs, item.Expr)+ }+ }+ }+ sourceColumns, err := q.mergeOmniExprSources(exprs...)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.SubqueryExpr:+ sourceColumns, isPlain, err := q.extractOmniSelectSourceColumns(e.Select)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: isPlain}, nil+ case *ast.ExistsExpr:+ sourceColumns, _, err := q.extractOmniSelectSourceColumns(e.Select)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.CaseExpr:+ var exprs []ast.ExprNode+ exprs = append(exprs, e.Operand)+ for _, when := range e.Whens {+ if when == nil {+ continue+ }+ exprs = append(exprs, when.Cond, when.Result)+ }+ exprs = append(exprs, e.Default)+ sourceColumns, err := q.mergeOmniExprSources(exprs...)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.BetweenExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr, e.Low, e.High)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.InExpr:+ exprs := append([]ast.ExprNode{e.Expr}, e.List...)+ sourceColumns, err := q.mergeOmniExprSources(exprs...)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ if e.Select != nil {+ selectSourceColumns, _, err := q.extractOmniSelectSourceColumns(e.Select)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ sourceColumns, _ = base.MergeSourceColumnSet(sourceColumns, selectSourceColumns)+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.LikeExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr, e.Pattern, e.Escape)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.IsExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.CastExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.ExtractExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.IntervalExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Value)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.CollateExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.MatchExpr:+ exprs := make([]ast.ExprNode, 0, len(e.Columns)+1)+ for _, column := range e.Columns {+ exprs = append(exprs, column)+ }+ exprs = append(exprs, e.Against)+ sourceColumns, err := q.mergeOmniExprSources(exprs...)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.ConvertExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Expr)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.DefaultExpr:+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: base.SourceColumnSet{}, IsPlainField: true}, nil+ case *ast.RowExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Items...)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ case *ast.MemberOfExpr:+ sourceColumns, err := q.mergeOmniExprSources(e.Value, e.Array)+ if err != nil {+ return base.QuerySpanResult{}, err+ }+ return base.QuerySpanResult{Name: q.omniExprName(e), SourceColumns: sourceColumns, IsPlainField: false}, nil+ default:+ return base.QuerySpanResult{}, errors.Errorf("unsupported omni MySQL expression %T", expr)+ }+}++func (q *omniQuerySpanExtractor) extractOmniSelectSourceColumns(selectStmt *ast.SelectStmt) (base.SourceColumnSet, bool, error) {+ subqueryExtractor := q.cloneOmniForSubquery()+ tableSource, err := subqueryExtractor.extractOmniSelectStmt(selectStmt)+ if err != nil {+ return nil, false, err+ }+ spanResults := tableSource.GetQuerySpanResult()+ sourceColumns := make(base.SourceColumnSet)+ for _, field := range spanResults {+ sourceColumns, _ = base.MergeSourceColumnSet(sourceColumns, field.SourceColumns)+ }+ isPlain := false+ if len(spanResults) == 1 {+ isPlain = spanResults[0].IsPlainField+ }+ return sourceColumns, isPlain, nil+}++func (q *omniQuerySpanExtractor) mergeOmniExprSources(exprs ...ast.ExprNode) (base.SourceColumnSet, error) {+ result := make(base.SourceColumnSet)+ for _, expr := range exprs {+ if expr == nil {+ continue+ }+ field, err := q.extractOmniExpr(expr)+ if err != nil {+ return nil, err+ }+ result, _ = base.MergeSourceColumnSet(result, field.SourceColumns)+ }+ return result, nil+}++func (q *omniQuerySpanExtractor) extractOmniColumnWild(columnRef *ast.ColumnRef, fromSources []base.TableSource) ([]base.QuerySpanResult, error) {+ if columnRef == nil {+ return nil, nil+ }+ return q.extractOmniTableWild(columnRef.Schema, columnRef.Table, fromSources)+}++func (q *omniQuerySpanExtractor) extractOmniTableWild(databaseName, tableName string, fromSources []base.TableSource) ([]base.QuerySpanResult, error) {+ if databaseName == "" && tableName == "" {+ var result []base.QuerySpanResult+ for _, fromSource := range fromSources {+ result = append(result, fromSource.GetQuerySpanResult()...)+ }+ return result, nil+ }++ result, ok := q.getAllTableColumnSources(databaseName, tableName)+ if !ok {+ return nil, &base.ResourceNotFoundError{+ Database: &databaseName,+ Table: &tableName,+ }+ }+ return result, nil+}++func (q *omniQuerySpanExtractor) omniExprName(expr ast.ExprNode) string {+ switch e := expr.(type) {+ case *ast.ColumnRef:+ var parts []string+ if e.Schema != "" {+ parts = append(parts, e.Schema)+ }+ if e.Table != "" {+ parts = append(parts, e.Table)+ }+ if e.Star {+ parts = append(parts, "*")+ } else {+ parts = append(parts, e.Column)+ }+ return strings.Join(parts, ".")+ case *ast.BinaryExpr:+ return q.omniExprName(e.Left) + omniBinaryOpName(e) + q.omniExprName(e.Right)+ case *ast.InExpr:+ var list []string+ for _, expr := range e.List {+ list = append(list, q.omniExprName(expr))+ }+ operator := " in "+ if e.Not {+ operator = " not in "+ }+ return q.omniExprName(e.Expr) + operator + "(" + strings.Join(list, ", ") + ")"+ case *ast.IntLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return strconv.FormatInt(e.Value, 10)+ case *ast.FloatLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return e.Value+ case *ast.StringLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return e.Value+ case *ast.BoolLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return strconv.FormatBool(e.Value)+ case *ast.NullLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return "NULL"+ case *ast.HexLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return e.Value+ case *ast.BitLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return e.Value+ case *ast.TemporalLit:+ if name := q.omniSlice(e.Loc); name != "" {+ return name+ }+ return e.Value+ default:+ if loc, ok := omniNodeLoc(e); ok {+ return q.omniSlice(loc)+ }+ return ""+ }+}++func omniBinaryOpName(expr *ast.BinaryExpr) string {+ if expr == nil {+ return ""+ }+ if expr.OriginalOp != "" {+ return expr.OriginalOp+ }+ return omniBinaryOpNames[expr.Op]+}++var omniBinaryOpNames = map[ast.BinaryOp]string{+ ast.BinOpAdd: "+",+ ast.BinOpSub: "-",+ ast.BinOpMul: "*",+ ast.BinOpDiv: "/",+ ast.BinOpMod: "%",+ ast.BinOpEq: "=",+ ast.BinOpNe: "!=",+ ast.BinOpLt: "<",+ ast.BinOpGt: ">",+ ast.BinOpLe: "<=",+ ast.BinOpGe: ">=",+ ast.BinOpAnd: " and ",+ ast.BinOpOr: " or ",+ ast.BinOpXor: " xor ",+ ast.BinOpBitAnd: "&",+ ast.BinOpBitOr: "|",+ ast.BinOpBitXor: "^",+ ast.BinOpShiftLeft: "<<",+ ast.BinOpShiftRight: ">>",+ ast.BinOpDivInt: " div ",+ ast.BinOpRegexp: " regexp ",+ ast.BinOpLikeEscape: " like ",+ ast.BinOpNullSafeEq: "<=>",+ ast.BinOpAssign: ":=",+ ast.BinOpJsonExtract: "->",+ ast.BinOpJsonUnquote: "->>",+ ast.BinOpSoundsLike: " sounds like ",+}++func (q *omniQuerySpanExtractor) omniSlice(loc ast.Loc) string {+ if loc.Start < 0 || loc.End <= loc.Start || loc.End > len(q.source) {+ return ""+ }+ return strings.TrimSpace(q.source[loc.Start:loc.End])+}++func omniNodeLoc(node ast.Node) (ast.Loc, bool) {+ if loc, ok := omniExprNodeLoc(node); ok {+ return loc, true+ }+ return omniLiteralNodeLoc(node)+}++func omniExprNodeLoc(node ast.Node) (ast.Loc, bool) {+ switch n := node.(type) {+ case *ast.ResTarget:+ return n.Loc, true+ case *ast.ColumnRef:+ return n.Loc, true+ case *ast.BinaryExpr:+ return n.Loc, true+ case *ast.UnaryExpr:+ return n.Loc, true+ case *ast.FuncCallExpr:+ return n.Loc, true+ case *ast.SubqueryExpr:+ return n.Loc, true+ case *ast.CaseExpr:+ return n.Loc, true+ case *ast.BetweenExpr:+ return n.Loc, true+ case *ast.InExpr:+ return n.Loc, true+ case *ast.LikeExpr:+ return n.Loc, true+ case *ast.IsExpr:+ return n.Loc, true+ case *ast.ExistsExpr:+ return n.Loc, true+ case *ast.CastExpr:+ return n.Loc, true+ case *ast.ExtractExpr:+ return n.Loc, true+ case *ast.IntervalExpr:+ return n.Loc, true+ case *ast.CollateExpr:+ return n.Loc, true+ case *ast.MatchExpr:+ return n.Loc, true+ case *ast.ConvertExpr:+ return n.Loc, true+ case *ast.DefaultExpr:+ return n.Loc, true+ case *ast.RowExpr:+ return n.Loc, true+ case *ast.MemberOfExpr:+ return n.Loc, true+ case *ast.ParenExpr:+ return n.Loc, true+ case *ast.StarExpr:+ return n.Loc, true+ default:+ return ast.Loc{}, false+ }+}++func omniLiteralNodeLoc(node ast.Node) (ast.Loc, bool) {+ switch n := node.(type) {+ case *ast.IntLit:+ return n.Loc, true+ case *ast.FloatLit:+ return n.Loc, true+ case *ast.StringLit:+ return n.Loc, true+ case *ast.BoolLit:+ return n.Loc, true+ case *ast.NullLit:+ return n.Loc, true+ case *ast.HexLit:+ return n.Loc, true+ case *ast.BitLit:+ return n.Loc, true+ case *ast.TemporalLit:+ return n.Loc, true+ case *ast.VariableRef:+ return n.Loc, true+ default:+ return ast.Loc{}, false+ }+}++func classifyOmniQueryType(node ast.Node, allSystems bool, source string) base.QueryType {+ switch n := node.(type) {+ case *ast.SelectStmt, *ast.TableStmt, *ast.ValuesStmt:+ if allSystems {+ return base.SelectInfoSchema+ }+ return base.Select+ case *ast.ExplainStmt:+ if isOmniDescribeStatement(source, n.Loc) {+ return base.SelectInfoSchema+ }+ if n.Analyze {+ switch n.Stmt.(type) {+ case *ast.SelectStmt, *ast.TableStmt, *ast.ValuesStmt:+ return base.Select+ default:+ return base.DML+ }+ }+ return base.Explain+ case *ast.ShowStmt:+ return base.SelectInfoSchema+ case *ast.SetStmt, *ast.SetDefaultRoleStmt, *ast.SetResourceGroupStmt, *ast.SetRoleStmt, *ast.SetTransactionStmt:+ return base.Select+ case *ast.SetPasswordStmt, *ast.HelpStmt:+ return base.QueryTypeUnknown+ case *ast.CreateDatabaseStmt, *ast.CreateTableStmt, *ast.CreateIndexStmt, *ast.CreateViewStmt,+ *ast.CreateEventStmt, *ast.CreateTriggerStmt, *ast.CreateFunctionStmt,+ *ast.CreateUserStmt, *ast.CreateRoleStmt, *ast.CreateTablespaceStmt, *ast.CreateServerStmt,+ *ast.CreateLogfileGroupStmt, *ast.CreateSpatialRefSysStmt, *ast.CreateResourceGroupStmt,+ *ast.AlterDatabaseStmt, *ast.AlterTableStmt, *ast.AlterViewStmt, *ast.AlterEventStmt,+ *ast.AlterUserStmt, *ast.AlterRoutineStmt, *ast.AlterTablespaceStmt, *ast.AlterServerStmt,+ *ast.AlterLogfileGroupStmt, *ast.AlterResourceGroupStmt, *ast.AlterInstanceStmt,+ *ast.DropDatabaseStmt, *ast.DropTableStmt, *ast.DropIndexStmt, *ast.DropViewStmt,+ *ast.DropEventStmt, *ast.DropTriggerStmt, *ast.DropRoutineStmt,+ *ast.DropUserStmt, *ast.DropRoleStmt, *ast.DropTablespaceStmt, *ast.DropServerStmt,+ *ast.DropLogfileGroupStmt, *ast.DropSpatialRefSysStmt, *ast.DropResourceGroupStmt,+ *ast.RenameTableStmt, *ast.RenameUserStmt, *ast.TruncateStmt, *ast.ImportTableStmt,+ *ast.GrantStmt, *ast.RevokeStmt, *ast.GrantRoleStmt, *ast.RevokeRoleStmt,+ *ast.InstallPluginStmt, *ast.UninstallPluginStmt, *ast.InstallComponentStmt,+ *ast.UninstallComponentStmt, *ast.CloneStmt:+ return base.DDL+ case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt, *ast.BeginStmt, *ast.CommitStmt,+ *ast.RollbackStmt, *ast.SavepointStmt, *ast.LockTablesStmt, *ast.UnlockTablesStmt,+ *ast.LoadDataStmt, *ast.PrepareStmt, *ast.ExecuteStmt, *ast.DeallocateStmt,+ *ast.CallStmt, *ast.DoStmt, *ast.HandlerOpenStmt, *ast.HandlerReadStmt, *ast.HandlerCloseStmt,+ *ast.AnalyzeTableStmt, *ast.OptimizeTableStmt, *ast.CheckTableStmt, *ast.RepairTableStmt,+ *ast.FlushStmt, *ast.KillStmt, *ast.ShutdownStmt, *ast.RestartStmt, *ast.XAStmt,+ *ast.SignalStmt, *ast.ResignalStmt, *ast.GetDiagnosticsStmt, *ast.DeclareVarStmt,+ *ast.DeclareConditionStmt, *ast.DeclareHandlerStmt, *ast.DeclareCursorStmt,+ *ast.IfStmt, *ast.WhileStmt, *ast.RepeatStmt, *ast.LoopStmt, *ast.LeaveStmt,+ *ast.IterateStmt, *ast.ReturnStmt, *ast.OpenCursorStmt, *ast.FetchCursorStmt,+ *ast.CloseCursorStmt, *ast.LockInstanceStmt, *ast.UnlockInstanceStmt, *ast.BinlogStmt,+ *ast.CacheIndexStmt, *ast.LoadIndexIntoCacheStmt, *ast.ResetPersistStmt,+ *ast.ChangeReplicationSourceStmt, *ast.ChangeReplicationFilterStmt,+ *ast.StartReplicaStmt, *ast.StopReplicaStmt, *ast.ResetReplicaStmt,+ *ast.PurgeBinaryLogsStmt, *ast.ResetMasterStmt,+ *ast.StartGroupReplicationStmt, *ast.StopGroupReplicationStmt:+ return base.DML+ default:+ return base.QueryTypeUnknown+ }+}++func isOmniDescribeStatement(source string, loc ast.Loc) bool {+ statement := strings.TrimSpace(source)+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(source) {+ statement = strings.TrimSpace(source[loc.Start:loc.End])+ }+ upper := strings.ToUpper(statement)+ fields := strings.Fields(upper)+ return len(fields) > 0 && (fields[0] == "DESCRIBE" || fields[0] == "DESC")+}++func collectOmniAccessTables(root ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) base.SourceColumnSet {+ result := make(base.SourceColumnSet)+ collectOmniAccessTablesFromNode(result, root, defaultDatabase, normalizeStarRocksCluster)+ return result+}++func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) {+ switch n := node.(type) {+ case nil:+ case *ast.SelectStmt:+ if n == nil {+ return+ }+ for _, cte := range n.CTEs {+ if cte != nil {+ collectOmniAccessTablesFromNode(result, cte.Select, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ if n.Left != nil {+ collectOmniAccessTablesFromNode(result, n.Left, defaultDatabase, normalizeStarRocksCluster)+ }+ if n.Right != nil {+ collectOmniAccessTablesFromNode(result, n.Right, defaultDatabase, normalizeStarRocksCluster)+ }+ for _, target := range n.TargetList {+ collectOmniAccessTablesFromExpr(result, target, defaultDatabase, normalizeStarRocksCluster)+ }+ for _, tableExpr := range n.From {+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ for _, groupBy := range n.GroupBy {+ collectOmniAccessTablesFromExpr(result, groupBy, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, n.Having, defaultDatabase, normalizeStarRocksCluster)+ for _, orderBy := range n.OrderBy {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.ExplainStmt:+ collectOmniAccessTablesFromNode(result, n.Stmt, defaultDatabase, normalizeStarRocksCluster)+ case *ast.InsertStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ for _, row := range n.Values {+ for _, expr := range row {+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.TableSource, defaultDatabase, normalizeStarRocksCluster)+ for _, assignment := range n.SetList {+ if assignment != nil {+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ for _, assignment := range n.OnDuplicateKey {+ if assignment != nil {+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.UpdateStmt:+ for _, tableExpr := range n.Tables {+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ }+ for _, assignment := range n.SetList {+ if assignment != nil {+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ for _, orderBy := range n.OrderBy {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.DeleteStmt:+ for _, tableExpr := range n.Tables {+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ }+ for _, tableExpr := range n.Using {+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ for _, orderBy := range n.OrderBy {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.CreateTableStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Like, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CreateIndexStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ for _, column := range n.Columns {+ if column != nil {+ collectOmniAccessTablesFromExpr(result, column.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.CreateViewStmt:+ collectOmniAccessTablesFromTableRef(result, n.Name, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.AlterTableStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ for _, command := range n.Commands {+ if command == nil {+ continue+ }+ collectOmniAccessTablesFromExpr(result, command.DefaultExpr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, command.ExchangeTable, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromOrderBy(result, command.OrderByItems, defaultDatabase, normalizeStarRocksCluster)+ }+ case *ast.DropTableStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.DropIndexStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ case *ast.DropViewStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Views, defaultDatabase, normalizeStarRocksCluster)+ case *ast.RenameTableStmt:+ for _, pair := range n.Pairs {+ if pair == nil {+ continue+ }+ collectOmniAccessTablesFromTableRef(result, pair.Old, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, pair.New, defaultDatabase, normalizeStarRocksCluster)+ }+ case *ast.TruncateStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.AnalyzeTableStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.OptimizeTableStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CheckTableStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.RepairTableStmt:+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ case *ast.TableStmt:+ if n == nil {+ return+ }+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)+ case *ast.ValuesStmt:+ for _, row := range n.Rows {+ for _, expr := range row {+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CallStmt:+ for _, arg := range n.Args {+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)+ }+ case *ast.SetStmt:+ for _, assignment := range n.Assignments {+ if assignment != nil {+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ case *ast.DoStmt:+ for _, expr := range n.Exprs {+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ }+ case *ast.HandlerOpenStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ case *ast.HandlerReadStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ case *ast.HandlerCloseStmt:+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ case *ast.LockTablesStmt:+ for _, table := range n.Tables {+ if table != nil {+ collectOmniAccessTablesFromTableRef(result, table.Table, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ default:+ }+}++func collectOmniAccessTablesFromOrderBy(result base.SourceColumnSet, orderByItems []*ast.OrderByItem, defaultDatabase string, normalizeStarRocksCluster bool) {+ for _, orderBy := range orderByItems {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+}++func collectOmniAccessTablesFromLimit(result base.SourceColumnSet, limit *ast.Limit, defaultDatabase string, normalizeStarRocksCluster bool) {+ if limit == nil {+ return+ }+ collectOmniAccessTablesFromExpr(result, limit.Count, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, limit.Offset, defaultDatabase, normalizeStarRocksCluster)+}++func collectOmniAccessTablesFromTableRefs(result base.SourceColumnSet, tableRefs []*ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {+ for _, tableRef := range tableRefs {+ collectOmniAccessTablesFromTableRef(result, tableRef, defaultDatabase, normalizeStarRocksCluster)+ }+}++func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {+ if tableRef == nil {+ return+ }+ if isOmniDualTable(tableRef) {+ return+ }+ database := tableRef.Schema+ if database == "" {+ database = defaultDatabase+ } else if normalizeStarRocksCluster {+ database = filterStarRocksClusterName(database)+ }+ result[base.ColumnResource{+ Database: database,+ Table: tableRef.Name,+ }] = true+}++func isOmniDualTable(tableRef *ast.TableRef) bool {+ return tableRef != nil && tableRef.Schema == "" && strings.EqualFold(tableRef.Name, "dual")+}++func collectOmniAccessTablesFromTableExpr(result base.SourceColumnSet, tableExpr ast.TableExpr, defaultDatabase string, normalizeStarRocksCluster bool) {+ switch n := tableExpr.(type) {+ case *ast.TableRef:+ collectOmniAccessTablesFromTableRef(result, n, defaultDatabase, normalizeStarRocksCluster)+ case *ast.JoinClause:+ collectOmniAccessTablesFromTableExpr(result, n.Left, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, n.Right, defaultDatabase, normalizeStarRocksCluster)+ switch condition := n.Condition.(type) {+ case *ast.OnCondition:+ collectOmniAccessTablesFromExpr(result, condition.Expr, defaultDatabase, normalizeStarRocksCluster)+ default:+ }+ case *ast.SubqueryExpr:+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.JsonTableExpr:+ collectOmniAccessTablesFromExpr(result, n.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Path, defaultDatabase, normalizeStarRocksCluster)+ default:+ }+}++func collectOmniAccessTablesFromExpr(result base.SourceColumnSet, expr ast.ExprNode, defaultDatabase string, normalizeStarRocksCluster bool) {+ switch e := expr.(type) {+ case nil:+ case *ast.ResTarget:+ collectOmniAccessTablesFromExpr(result, e.Val, defaultDatabase, normalizeStarRocksCluster)+ case *ast.BinaryExpr:+ collectOmniAccessTablesFromExpr(result, e.Left, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Right, defaultDatabase, normalizeStarRocksCluster)+ case *ast.UnaryExpr:+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)+ case *ast.FuncCallExpr:+ for _, arg := range e.Args {+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, e.Separator, defaultDatabase, normalizeStarRocksCluster)+ for _, orderBy := range e.OrderBy {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ if e.Over != nil {+ for _, partitionBy := range e.Over.PartitionBy {+ collectOmniAccessTablesFromExpr(result, partitionBy, defaultDatabase, normalizeStarRocksCluster)+ }+ for _, orderBy := range e.Over.OrderBy {+ if orderBy != nil {+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ }+ }+ }+ case *ast.SubqueryExpr:+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CaseExpr:+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)+ for _, when := range e.Whens {+ if when == nil {+ continue+ }+ collectOmniAccessTablesFromExpr(result, when.Cond, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, when.Result, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, e.Default, defaultDatabase, normalizeStarRocksCluster)+ case *ast.BetweenExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Low, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.High, defaultDatabase, normalizeStarRocksCluster)+ case *ast.InExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ for _, item := range e.List {+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.LikeExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Pattern, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Escape, defaultDatabase, normalizeStarRocksCluster)+ case *ast.IsExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ case *ast.ExistsExpr:+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CastExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ case *ast.ExtractExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ case *ast.IntervalExpr:+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)+ case *ast.CollateExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ case *ast.MatchExpr:+ for _, column := range e.Columns {+ collectOmniAccessTablesFromExpr(result, column, defaultDatabase, normalizeStarRocksCluster)+ }+ collectOmniAccessTablesFromExpr(result, e.Against, defaultDatabase, normalizeStarRocksCluster)+ case *ast.ConvertExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ case *ast.RowExpr:+ for _, item := range e.Items {+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)+ }+ case *ast.MemberOfExpr:+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Array, defaultDatabase, normalizeStarRocksCluster)+ case *ast.ParenExpr:+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ default:+ }+}++func filterStarRocksClusterName(databaseName string) string {+ list := strings.Split(databaseName, ":")+ if len(list) > 1 {+ return list[len(list)-1]+ }+ return databaseName+}diff --git a/backend/plugin/parser/mysql/query_span_extractor_omni_test.go b/backend/plugin/parser/mysql/query_span_extractor_omni_test.gonew file mode 100644index 0000000000..b090255eac--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_extractor_omni_test.go@@ -0,0 +1,1563 @@+package mysql++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestOmniQuerySpanScaffold_QueryTypesAndAccessTables(t *testing.T) {+ tests := []struct {+ name string+ statement string+ wantType base.QueryType+ wantResources []base.ColumnResource+ }{+ {+ name: "explain_analyze_select_accesses_table",+ statement: "EXPLAIN ANALYZE SELECT * FROM t",+ wantType: base.Select,+ wantResources: []base.ColumnResource{+ {Database: "db", Table: "t"},+ },+ },+ {+ name: "set_is_select_with_no_access_tables",+ statement: "SET CHARSET DEFAULT",+ wantType: base.Select,+ },+ {+ name: "show_is_select_info_schema",+ statement: "SHOW DATABASES",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "plain_explain_is_explain",+ statement: "EXPLAIN SELECT * FROM t",+ wantType: base.Explain,+ },+ {+ name: "describe_is_info_schema",+ statement: "DESCRIBE t",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "desc_tab_is_info_schema",+ statement: "DESC\t t",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "desc_newline_is_info_schema",+ statement: "DESC\n t",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "ddl",+ statement: "CREATE TABLE t(a INT)",+ wantType: base.DDL,+ },+ {+ name: "dml",+ statement: "INSERT INTO t VALUES(1)",+ wantType: base.DML,+ },+ {+ name: "system_select",+ statement: "SELECT * FROM mysql.user",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "standard_select_accesses_table",+ statement: "SELECT * FROM t WHERE a > 0",+ wantType: base.Select,+ wantResources: []base.ColumnResource{+ {Database: "db", Table: "t"},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.wantType, span.Type)+ require.Equal(t, sourceColumnSetFromResources(tc.wantResources), span.SourceColumns)+ if tc.wantType != base.Select {+ require.Empty(t, span.Results)+ }+ require.Empty(t, span.PredicateColumns)+ })+ }+}++func TestOmniQuerySpanPhase1_SimpleSelectResultColumns(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ }{+ {+ name: "constant",+ statement: "SELECT 1",+ want: []base.QuerySpanResult{+ {Name: "1", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ },+ },+ {+ name: "bare_column",+ statement: "SELECT a FROM t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "alias",+ statement: "SELECT a AS x FROM t",+ want: []base.QuerySpanResult{+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "qualified_columns",+ statement: "SELECT a, t.b, db.t.c FROM t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ {+ name: "star",+ statement: "SELECT * FROM t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ {+ name: "star_and_column",+ statement: "SELECT *, a FROM t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "table_star",+ statement: "SELECT t.* FROM t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ }++ gCtx := newOmniTestQuerySpanContext()+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Equal(t, tc.want, span.Results)+ })+ }+}++func TestOmniQuerySpanPhase2_ExpressionSourceMerging(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ "SELECT max(a), a-b AS c1, a=b AS c2, a>b, b in (a, c) FROM t",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {+ Name: "max(a)",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}),+ IsPlainField: false,+ },+ {+ Name: "c1",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t", Column: "a"},+ {Database: "db", Table: "t", Column: "b"},+ }),+ IsPlainField: false,+ },+ {+ Name: "c2",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t", Column: "a"},+ {Database: "db", Table: "t", Column: "b"},+ }),+ IsPlainField: false,+ },+ {+ Name: "a>b",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t", Column: "a"},+ {Database: "db", Table: "t", Column: "b"},+ }),+ IsPlainField: false,+ },+ {+ Name: "b in (a, c)",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t", Column: "a"},+ {Database: "db", Table: "t", Column: "b"},+ {Database: "db", Table: "t", Column: "c"},+ }),+ IsPlainField: false,+ },+ }, span.Results)+}++func TestOmniQuerySpanPhase3_FromJoinAliasAndScope(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ }{+ {+ name: "table_alias",+ statement: "SELECT x.a FROM t AS x",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "join_on_with_qualified_star_and_column",+ statement: "SELECT t1.*, t2.c, 0 FROM t1 JOIN t2 ON 1 = 1",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "c"}}), IsPlainField: true},+ {Name: "0", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ },+ },+ {+ name: "join_using_merges_star_columns",+ statement: "SELECT * FROM t AS t1 JOIN t AS t2 USING(a)",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ }++ gCtx := newOmniTestQuerySpanContext()+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ })+ }+}++func TestOmniQuerySpanPhase4_SubqueryExpressions(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("scalar_subquery_results", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT 1 AS col_1, (SELECT(2)) AS col_2, (SELECT AVG(a + b * c) FROM t) AS avg_a_b_c FROM t",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "col_1", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ {Name: "col_2", SourceColumns: base.SourceColumnSet{}, IsPlainField: false},+ {+ Name: "avg_a_b_c",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t", Column: "a"},+ {Database: "db", Table: "t", Column: "b"},+ {Database: "db", Table: "t", Column: "c"},+ }),+ IsPlainField: false,+ },+ }, span.Results)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t"}}), span.SourceColumns)+ })++ t.Run("correlated_subquery_access_tables", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total_paintings FROM galleries g",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "city", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "galleries", Column: "city"}}), IsPlainField: true},+ {Name: "total_paintings", SourceColumns: base.SourceColumnSet{}, IsPlainField: false},+ }, span.Results)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "galleries"},+ {Database: "db", Table: "paintings"},+ }), span.SourceColumns)+ })+}++func TestOmniQuerySpanPhase5And6_CTEAndSetOperations(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ }{+ {+ name: "simple_cte",+ statement: "WITH t1 AS (SELECT * FROM t) SELECT * FROM t1",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ {+ name: "cte_column_aliases",+ statement: "WITH t1(x, y, z) AS (SELECT * FROM t) SELECT * FROM t1",+ want: []base.QuerySpanResult{+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "y", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "z", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ },+ {+ name: "union_merges_positionally",+ statement: "SELECT 1 AS c1 UNION SELECT a FROM t",+ want: []base.QuerySpanResult{+ {Name: "c1", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: false},+ },+ },+ {+ name: "recursive_keyword_non_recursive_cte",+ statement: "WITH RECURSIVE t1 AS (SELECT 1 AS c1 UNION SELECT a FROM t) SELECT * FROM t1",+ want: []base.QuerySpanResult{+ {Name: "c1", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: false},+ },+ },+ }++ gCtx := newOmniTestQuerySpanContext()+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ })+ }+}++func TestOmniQuerySpanPhase7_JSONTable(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ `SELECT *+FROM products,+JSON_TABLE(product_info, '$' COLUMNS (+ product_id INT PATH '$.id',+ product_name VARCHAR(50) PATH '$.name'+)) AS jt`,+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "product_info", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ {Name: "product_id", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ {Name: "product_name", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ }, span.Results)+}++func TestOmniQuerySpanPhase9_ResourceNotFoundFailOpen(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{Name: "db"}+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ span, err := newOmniQuerySpanExtractor("db", base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ Engine: storepb.Engine_MYSQL,+ }, false).getOmniQuerySpan(context.Background(), "SELECT * FROM t WHERE a > 0")+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t"}}), span.SourceColumns)+ require.Empty(t, span.Results)+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))+}++func TestOmniQuerySpanSystematicMigrationRegressions(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("derived_table_column_aliases_are_applied", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT x.c1 FROM (SELECT a, b FROM t) AS x(c1, c2)",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "c1", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("in_subquery_sources_are_part_of_result_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT a IN (SELECT t2.c FROM t2) AS matched FROM t1",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {+ Name: "matched",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t1", Column: "a"},+ {Database: "db", Table: "t2", Column: "c"},+ }),+ IsPlainField: false,+ },+ }, span.Results)+ })++ t.Run("explicit_expression_nodes_do_not_drop_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ `SELECT+ EXISTS (SELECT t2.a FROM t2 WHERE t2.a = t.a) AS exists_field,+ CONVERT(a USING utf8mb4) AS converted,+ a COLLATE utf8mb4_bin AS collated,+ MATCH(a) AGAINST (b) AS matched,+ ROW(a, b) AS row_field,+ a MEMBER OF(c) AS member_field,+ a + INTERVAL b DAY AS interval_field+FROM t`,+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {+ Name: "exists_field",+ SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{+ {Database: "db", Table: "t2", Column: "a"},+ }),+ IsPlainField: false,+ },+ {Name: "converted", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: false},+ {Name: "collated", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: false},+ {Name: "matched", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "b"}}), IsPlainField: false},+ {Name: "row_field", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "b"}}), IsPlainField: false},+ {Name: "member_field", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "c"}}), IsPlainField: false},+ {Name: "interval_field", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "b"}}), IsPlainField: false},+ }, span.Results)+ })++ t.Run("legacy_dml_roots_stay_dml", func(t *testing.T) {+ tests := []string{+ "CALL p()",+ "DO 1",+ "HANDLER t OPEN",+ "HANDLER t READ FIRST",+ "HANDLER t CLOSE",+ }+ for _, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ require.Equal(t, base.DML, span.Type, statement)+ require.Empty(t, span.Results, statement)+ }+ })++ t.Run("table_and_values_roots_return_select_results", func(t *testing.T) {+ tableSpan, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "TABLE t")+ require.NoError(t, err)+ require.Equal(t, base.Select, tableSpan.Type)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ }, tableSpan.Results)++ valuesSpan, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "VALUES ROW(1, 2 + 3)")+ require.NoError(t, err)+ require.Equal(t, base.Select, valuesSpan.Type)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "1", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ {Name: "2+3", SourceColumns: base.SourceColumnSet{}, IsPlainField: false},+ }, valuesSpan.Results)+ })+}++func TestOmniQuerySpanScenarioBatch1_RootAndQueryTypeCoverage(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("select_family_roots", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ wantType base.QueryType+ wantResults []base.QuerySpanResult+ wantResources []base.ColumnResource+ }{+ {+ name: "values_default",+ statement: "VALUES ROW(DEFAULT)",+ wantType: base.Select,+ wantResults: []base.QuerySpanResult{+ {Name: "DEFAULT", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ },+ },+ {+ name: "explain_analyze_table",+ statement: "EXPLAIN ANALYZE TABLE t",+ wantType: base.Select,+ wantResources: []base.ColumnResource{{Database: "db", Table: "t"}},+ },+ {+ name: "explain_analyze_values",+ statement: "EXPLAIN ANALYZE VALUES ROW(1, 2 + 3)",+ wantType: base.Select,+ },+ {+ name: "explain_analyze_insert",+ statement: "EXPLAIN ANALYZE INSERT INTO t VALUES(1, 2, 3)",+ wantType: base.DML,+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.wantType, span.Type)+ require.Equal(t, tc.wantResults, span.Results)+ require.Equal(t, sourceColumnSetFromResources(tc.wantResources), span.SourceColumns)+ })+ }+ })++ t.Run("multiple_statements_are_rejected", func(t *testing.T) {+ _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT 1; SELECT 2;")+ require.ErrorContains(t, err, "expected exactly 1 statement")+ })++ t.Run("ddl_bucket_roots", func(t *testing.T) {+ tests := []string{+ "CREATE DATABASE d",+ "CREATE VIEW v AS SELECT a FROM t",+ "ALTER TABLE t ADD COLUMN d INT",+ "DROP TABLE t",+ "RENAME TABLE t TO t_new",+ "TRUNCATE TABLE t",+ }+ for _, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ require.Equal(t, base.DDL, span.Type, statement)+ require.Empty(t, span.Results, statement)+ }+ })++ t.Run("dml_bucket_roots", func(t *testing.T) {+ tests := []string{+ "UPDATE t SET a = 1",+ "DELETE FROM t WHERE a = 1",+ "BEGIN",+ "COMMIT",+ "ROLLBACK",+ "SAVEPOINT s",+ "LOCK TABLES t READ",+ "UNLOCK TABLES",+ "PREPARE stmt FROM 'SELECT 1'",+ "EXECUTE stmt",+ "DEALLOCATE PREPARE stmt",+ }+ for _, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ require.Equal(t, base.DML, span.Type, statement)+ require.Empty(t, span.Results, statement)+ }+ })+}++func TestOmniQuerySpanScenarioBatch4_QueryTypeEdges(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("legacy_query_type_buckets", func(t *testing.T) {+ tests := []struct {+ statement string+ want base.QueryType+ }{+ {statement: "SET PASSWORD = 'new_password'", want: base.QueryTypeUnknown},+ {statement: "CREATE USER u IDENTIFIED BY 'p'", want: base.DDL},+ {statement: "ALTER USER u IDENTIFIED BY 'p2'", want: base.DDL},+ {statement: "DROP USER u", want: base.DDL},+ {statement: "GRANT SELECT ON *.* TO u", want: base.DDL},+ {statement: "REVOKE SELECT ON *.* FROM u", want: base.DDL},+ {statement: "IMPORT TABLE FROM '/tmp/t.sdi'", want: base.DDL},+ {statement: "REPLACE INTO t VALUES(1, 2, 3)", want: base.DML},+ {statement: "LOAD DATA INFILE '/tmp/t.csv' INTO TABLE t", want: base.DML},+ {statement: "CHANGE REPLICATION SOURCE TO SOURCE_HOST = '127.0.0.1'", want: base.DML},+ {statement: "START REPLICA", want: base.DML},+ {statement: "STOP REPLICA", want: base.DML},+ {statement: "RESET REPLICA", want: base.DML},+ {statement: "PURGE BINARY LOGS TO 'mysql-bin.010'", want: base.DML},+ {statement: "RESET MASTER", want: base.DML},+ {statement: "START GROUP_REPLICATION", want: base.DML},+ {statement: "STOP GROUP_REPLICATION", want: base.DML},+ {statement: "HELP 'contents'", want: base.QueryTypeUnknown},+ }+ for _, tc := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err, tc.statement)+ require.Equal(t, tc.want, span.Type, tc.statement)+ require.Empty(t, span.Results, tc.statement)+ }+ })+}++func TestOmniQuerySpanScenarioBatch1_ExpressionCoverage(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("quoted_identifier_names", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT `select`, `Camel` AS `ExactCase` FROM keyword_table",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "select", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "keyword_table", Column: "select"}}), IsPlainField: true},+ {Name: "ExactCase", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "keyword_table", Column: "Camel"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("variable_refs_have_empty_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT @x AS user_var, @@version AS system_var",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "user_var", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ {Name: "system_var", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ }, span.Results)+ })++ t.Run("json_extract_operator_uses_object_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT product_info->'$.id' AS id FROM products",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "id", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: false},+ }, span.Results)+ })++ t.Run("group_concat_separator_contributes_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT GROUP_CONCAT(a SEPARATOR b) AS gc FROM t",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "gc", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "b"}}), IsPlainField: false},+ }, span.Results)+ })++ t.Run("group_concat_order_by_contributes_lineage", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT GROUP_CONCAT(a ORDER BY b) AS gc FROM t",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "gc", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t", Column: "b"}}), IsPlainField: false},+ }, span.Results)+ })++ t.Run("duplicate_output_names_are_preserved", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT a AS x, b AS x FROM t",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ }, span.Results)+ })+}++func TestOmniQuerySpanScenarioBatch2_FromJoinAndScopeCoverage(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("table_source_variants", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ wantSrc []base.ColumnResource+ }{+ {+ name: "database_qualified_table",+ statement: "SELECT db.t.a FROM db.t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{{Database: "db", Table: "t"}},+ },+ {+ name: "parenthesized_single_table",+ statement: "SELECT a FROM (t)",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{{Database: "db", Table: "t"}},+ },+ {+ name: "comma_separated_tables",+ statement: "SELECT t1.a, t2.b FROM t1, t2",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{{Database: "db", Table: "t1"}, {Database: "db", Table: "t2"}},+ },+ {+ name: "dual",+ statement: "SELECT 1 FROM DUAL",+ want: []base.QuerySpanResult{+ {Name: "1", SourceColumns: base.SourceColumnSet{}, IsPlainField: true},+ },+ },+ {+ name: "dual_join_right",+ statement: "SELECT t.a FROM t JOIN DUAL",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{{Database: "db", Table: "t"}},+ },+ {+ name: "dual_join_left",+ statement: "SELECT t.a FROM DUAL JOIN t",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{{Database: "db", Table: "t"}},+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ require.Equal(t, sourceColumnSetFromResources(tc.wantSrc), span.SourceColumns)+ })+ }+ })++ t.Run("join_variants_expose_columns", func(t *testing.T) {+ tests := []string{+ "SELECT * FROM t1 STRAIGHT_JOIN t2",+ "SELECT * FROM t1 LEFT JOIN t2 ON t1.a = t2.a",+ "SELECT * FROM t1 RIGHT JOIN t2 ON t1.a = t2.a",+ "SELECT * FROM t1 NATURAL LEFT JOIN t2",+ "SELECT * FROM t1 NATURAL RIGHT JOIN t2",+ }+ want := []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}}), IsPlainField: true},+ }+ wantNonNatural := append([]base.QuerySpanResult{}, want...)+ wantNonNatural = append(wantNonNatural,+ base.QuerySpanResult{Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},+ base.QuerySpanResult{Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ base.QuerySpanResult{Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "c"}}), IsPlainField: true},+ )+ for i, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ if i < 3 {+ require.Equal(t, wantNonNatural, span.Results, statement)+ } else {+ require.Equal(t, want, span.Results, statement)+ }+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1"}, {Database: "db", Table: "t2"}}), span.SourceColumns, statement)+ }+ })++ t.Run("derived_table_edges", func(t *testing.T) {+ _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT * FROM (SELECT a, b FROM t) AS x(c1)",+ )+ require.ErrorContains(t, err, "derived table column list length")++ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT y.c1 FROM (SELECT x.c1 FROM (SELECT a FROM t) AS x(c1)) AS y",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "c1", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)++ span, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT x.a FROM t, LATERAL (SELECT t.a) AS x",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("join_dependency_edges", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ wantSrc []base.ColumnResource+ }{+ {+ name: "join_on_subquery_contributes_access_table",+ statement: "SELECT t1.a FROM t1 JOIN t2 ON EXISTS (SELECT t.a FROM t WHERE t.a = t2.a)",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{+ {Database: "db", Table: "t1"},+ {Database: "db", Table: "t2"},+ {Database: "db", Table: "t"},+ },+ },+ {+ name: "nested_join_tree_preserves_visible_order",+ statement: "SELECT * FROM t1 JOIN (t2 JOIN t ON t2.a = t.a) ON t1.a = t2.a",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "c"}}), IsPlainField: true},+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "c"}}), IsPlainField: true},+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ {Name: "c", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "c"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{+ {Database: "db", Table: "t1"},+ {Database: "db", Table: "t2"},+ {Database: "db", Table: "t"},+ },+ },+ {+ name: "parenthesized_table_reference_list",+ statement: "SELECT t1.a, t2.b FROM (t1, t2)",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ },+ wantSrc: []base.ColumnResource{+ {Database: "db", Table: "t1"},+ {Database: "db", Table: "t2"},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ require.Equal(t, sourceColumnSetFromResources(tc.wantSrc), span.SourceColumns)+ })+ }+ })+}++func TestOmniQuerySpanScenarioBatch2_SubqueryCTEAndSetCoverage(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("subquery_access_locations", func(t *testing.T) {+ tests := []string{+ "SELECT a FROM t WHERE EXISTS (SELECT t2.a FROM t2)",+ "SELECT a FROM t GROUP BY a HAVING EXISTS (SELECT t2.a FROM t2)",+ "SELECT a FROM t ORDER BY (SELECT t2.a FROM t2)",+ }+ for _, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t"}, {Database: "db", Table: "t2"}}), span.SourceColumns, statement)+ }+ })++ t.Run("cte_and_set_error_edges", func(t *testing.T) {+ _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "WITH c(x) AS (SELECT a, b FROM t) SELECT * FROM c",+ )+ require.ErrorContains(t, err, "CTE column list")++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "WITH RECURSIVE c(x) AS (SELECT a, b FROM t UNION SELECT a, b FROM t2) SELECT * FROM c",+ )+ require.ErrorContains(t, err, "different column counts")++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT a FROM t UNION SELECT a, b FROM t2",+ )+ require.ErrorContains(t, err, "UNION operator left has 1 fields, right has 2 fields")++ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "WITH c1 AS (SELECT a FROM c2), c2 AS (SELECT a FROM t) SELECT * FROM c1",+ )+ require.NoError(t, err)+ require.Empty(t, span.Results)+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))+ })++ t.Run("set_operation_in_derived_and_cte", func(t *testing.T) {+ tests := []string{+ "SELECT x.a FROM (SELECT a FROM t UNION SELECT a FROM t2) AS x",+ "WITH c AS (SELECT a FROM t UNION SELECT a FROM t2) SELECT a FROM c",+ }+ for _, statement := range tests {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), statement)+ require.NoError(t, err, statement)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},+ }, span.Results, statement)+ }+ })++ t.Run("additional_set_operation_edges", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ }{+ {+ name: "intersect",+ statement: "SELECT a FROM t INTERSECT SELECT a FROM t2",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}}), IsPlainField: false},+ },+ },+ {+ name: "except",+ statement: "SELECT a FROM t EXCEPT SELECT a FROM t2",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}}), IsPlainField: false},+ },+ },+ {+ name: "parenthesized_grouping",+ statement: "SELECT a FROM t UNION (SELECT a FROM t2 UNION SELECT b FROM t1)",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "a"}, {Database: "db", Table: "t1", Column: "b"}}), IsPlainField: false},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ })+ }+ })++ t.Run("scope_resolution_edges", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.QuerySpanResult+ }{+ {+ name: "nested_subquery_nearest_scope_alias",+ statement: "SELECT (SELECT x.a FROM t2 AS x WHERE x.b = outer_t.b) AS inner_a FROM t AS outer_t",+ want: []base.QuerySpanResult{+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "unqualified_inner_column_uses_legacy_resolution_order",+ statement: "SELECT (SELECT a FROM t2) AS inner_a FROM t1",+ want: []base.QuerySpanResult{+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "inner_alias_shadow_keeps_legacy_outer_first_lookup",+ statement: "SELECT (SELECT x.a FROM t2 AS x) AS inner_a FROM t1 AS x",+ want: []base.QuerySpanResult{+ {Name: "inner_a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t1", Column: "a"}}), IsPlainField: true},+ },+ },+ {+ name: "cte_name_shadows_physical_table",+ statement: "WITH t AS (SELECT b FROM t2) SELECT * FROM t",+ want: []base.QuerySpanResult{+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t2", Column: "b"}}), IsPlainField: true},+ },+ },+ {+ name: "nested_cte_visibility_order",+ statement: "WITH c1 AS (SELECT a FROM t), c2 AS (SELECT a FROM c1 UNION SELECT b FROM t2) SELECT * FROM c2",+ want: []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "b"}}), IsPlainField: false},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, span.Results)+ })+ }+ })++ t.Run("recursive_cte_reaches_stable_source_closure", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "WITH RECURSIVE c(x) AS (SELECT a FROM t UNION SELECT t2.b FROM t2 JOIN c ON c.x = t2.a) SELECT * FROM c",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "b"}}), IsPlainField: false},+ }, span.Results)+ })++ t.Run("recursive_cte_left_deep_set_op_registers_before_recursive_left_branch", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "WITH RECURSIVE c(x) AS (SELECT a FROM t UNION SELECT t2.b FROM t2 JOIN c ON c.x = t2.a UNION SELECT t2.c FROM t2 JOIN c ON c.x = t2.b) SELECT * FROM c",+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "x", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}, {Database: "db", Table: "t2", Column: "b"}, {Database: "db", Table: "t2", Column: "c"}}), IsPlainField: false},+ }, span.Results)+ })+}++func TestOmniQuerySpanScenarioBatch4_JSONTableCoverage(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ `SELECT *+FROM products,+JSON_TABLE(product_info, '$' COLUMNS (+ product_id INT PATH '$.id',+ NESTED PATH '$.variants[*]' COLUMNS (+ variant_id INT PATH '$.id',+ variant_name VARCHAR(50) PATH '$.name'+ )+)) AS jt`,+ )+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "product_info", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ {Name: "product_id", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ {Name: "variant_id", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ {Name: "variant_name", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ }, span.Results)++ _, err = newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ `SELECT *+FROM products,+JSON_TABLE(product_info, '$' COLUMNS (+ product_id INT PATH '$.id'+))`,+ )+ require.Error(t, err)+}++func TestOmniQuerySpanScenarioBatch4_AccessTableExpressions(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ tests := []struct {+ name string+ statement string+ want []base.ColumnResource+ }{+ {+ name: "function_argument_subquery",+ statement: "SELECT COALESCE((SELECT t2.a FROM t2), a) FROM t",+ want: []base.ColumnResource{+ {Database: "db", Table: "t"},+ {Database: "db", Table: "t2"},+ },+ },+ {+ name: "function_order_by_subquery",+ statement: "SELECT GROUP_CONCAT(a ORDER BY (SELECT t2.b FROM t2)) FROM t",+ want: []base.ColumnResource{+ {Database: "db", Table: "t"},+ {Database: "db", Table: "t2"},+ },+ },+ {+ name: "json_table_document_subquery",+ statement: "SELECT * FROM JSON_TABLE((SELECT product_info FROM products), '$' COLUMNS (product_id INT PATH '$.id')) AS jt",+ want: []base.ColumnResource{+ {Database: "db", Table: "products"},+ },+ },+ {+ name: "values_subquery",+ statement: "VALUES ROW((SELECT a FROM t))",+ want: []base.ColumnResource{+ {Database: "db", Table: "t"},+ },+ },+ {+ name: "set_assignment_subquery",+ statement: "SET @x = (SELECT a FROM t)",+ want: []base.ColumnResource{+ {Database: "db", Table: "t"},+ },+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, sourceColumnSetFromResources(tc.want), span.SourceColumns)+ })+ }+}++func TestOmniQuerySpanScenarioBatch3_ErrorAndMetadataCoverage(t *testing.T) {+ t.Run("error_semantics", func(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT a INTO @x FROM t")+ require.ErrorContains(t, err, "unsupported select statement with into")++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "TABLE t INTO @x")+ require.ErrorContains(t, err, "unsupported table statement with into")++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT FROM")+ require.Error(t, err)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT * FORM invalid_table;")+ require.Error(t, err)++ _, err = GetQuerySpan(context.Background(), gCtx, base.Statement{+ Text: "SELECT * FORM invalid_table;",+ Start: &storepb.Position{Line: 3, Column: 1},+ }, "db", "", false)+ var syntaxErr *base.SyntaxError+ require.ErrorAs(t, err, &syntaxErr)+ require.Equal(t, int32(3), syntaxErr.Position.Line)++ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "")+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Empty(t, span.Results)+ require.Empty(t, span.SourceColumns)++ span, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "EXPLAIN ANALYZE SELECT * FROM missing")+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Empty(t, span.Results)+ require.Nil(t, span.NotFoundError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "INSERT INTO t SELECT * FROM mysql.user")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "CREATE TABLE new_t AS SELECT * FROM mysql.user")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "EXPLAIN SELECT * FROM t, mysql.user")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "CALL p((SELECT a FROM t), (SELECT user FROM mysql.user))")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SET @x = (SELECT a FROM t), @y = (SELECT user FROM mysql.user)")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "TABLE t ORDER BY (SELECT user FROM mysql.user)")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "TABLE t LIMIT (SELECT COUNT(*) FROM mysql.user)")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "LOCK TABLES t READ, mysql.user READ")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "DROP TABLE t, mysql.user")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)++ _, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "RENAME TABLE t TO t_new, mysql.user TO mysql.user_new")+ require.ErrorIs(t, err, base.MixUserSystemTablesError)+ })++ t.Run("view_lineage", func(t *testing.T) {+ gCtx := newOmniViewTestQuerySpanContext()++ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT * FROM v")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "va", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ {Name: "b", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "b"}}), IsPlainField: true},+ }, span.Results)++ span, err = newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT * FROM v2")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "va", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("missing_view_dependency_fails_open", func(t *testing.T) {+ gCtx := newOmniViewTestQuerySpanContext()+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT * FROM broken_v")+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Empty(t, span.Results)+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))+ })++ t.Run("system_schema_detection", func(t *testing.T) {+ tests := []struct {+ name string+ statement string+ ignoreCaseSensitive bool+ wantType base.QueryType+ wantSource []base.ColumnResource+ wantNotFound bool+ }{+ {+ name: "information_schema_lowercase",+ statement: "SELECT * FROM information_schema.tables",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "information_schema_uppercase_ignore_case",+ statement: "SELECT * FROM INFORMATION_SCHEMA.tables",+ ignoreCaseSensitive: true,+ wantType: base.SelectInfoSchema,+ },+ {+ name: "information_schema_uppercase_case_sensitive",+ statement: "SELECT * FROM INFORMATION_SCHEMA.tables",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "performance_schema_lowercase",+ statement: "SELECT * FROM performance_schema.events_statements_current",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "mysql_lowercase",+ statement: "SELECT * FROM mysql.user",+ wantType: base.SelectInfoSchema,+ },+ {+ name: "mysql_uppercase_ignore_case",+ statement: "SELECT * FROM MYSQL.user",+ ignoreCaseSensitive: true,+ wantType: base.SelectInfoSchema,+ },+ {+ name: "mysql_uppercase_case_sensitive",+ statement: "SELECT * FROM MYSQL.user",+ wantType: base.Select,+ wantNotFound: true,+ wantSource: []base.ColumnResource{+ {Database: "MYSQL", Table: "user"},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), tc.ignoreCaseSensitive).getOmniQuerySpan(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.wantType, span.Type)+ require.Equal(t, sourceColumnSetFromResources(tc.wantSource), span.SourceColumns)+ if tc.wantNotFound {+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))+ } else {+ require.NoError(t, span.NotFoundError)+ }+ })+ }+ })+}++func TestOmniQuerySpanScenarioBatch4_CaseSensitivityCoverage(t *testing.T) {+ gCtx := newOmniTestQuerySpanContext()++ t.Run("table_alias_case_sensitivity", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT X.a FROM t AS x")+ require.NoError(t, err)+ require.Empty(t, span.Results)+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))++ span, err = newOmniQuerySpanExtractor("db", gCtx, true).getOmniQuerySpan(context.Background(), "SELECT X.a FROM t AS x")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("derived_table_alias_case_sensitivity", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT X.a FROM (SELECT a FROM t) AS x")+ require.NoError(t, err)+ require.Empty(t, span.Results)+ require.ErrorAs(t, span.NotFoundError, new(*base.ResourceNotFoundError))++ span, err = newOmniQuerySpanExtractor("db", gCtx, true).getOmniQuerySpan(context.Background(), "SELECT X.a FROM (SELECT a FROM t) AS x")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })++ t.Run("cte_name_case_sensitivity", func(t *testing.T) {+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "WITH c AS (SELECT a FROM t) SELECT C.a FROM c AS C")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ })+}++func TestOmniQuerySpanScenarioBatch5_StarRocksAndMetadataCoverage(t *testing.T) {+ t.Run("starrocks_cluster_qualified_tables", func(t *testing.T) {+ gCtx := newOmniStarRocksTestQuerySpanContext()+ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT `cluster_name:db`.products.product_info FROM `cluster_name:db`.products",+ )+ require.NoError(t, err)+ require.Equal(t, base.Select, span.Type)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "product_info", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},+ }, span.Results)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products"}}), span.SourceColumns)+ })++ t.Run("starrocks_system_user_mixing_uses_normalized_database", func(t *testing.T) {+ gCtx := newOmniStarRocksTestQuerySpanContext()+ _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(+ context.Background(),+ "SELECT * FROM products, `cluster_name:information_schema`.tables",+ )+ require.ErrorIs(t, err, base.MixUserSystemTablesError)+ })++ t.Run("mysql_and_starrocks_lineage_match_without_cluster_prefix", func(t *testing.T) {+ mysqlSpan, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ "SELECT product_info FROM products",+ )+ require.NoError(t, err)++ starrocksSpan, err := newOmniQuerySpanExtractor("db", newOmniStarRocksTestQuerySpanContext(), false).getOmniQuerySpan(+ context.Background(),+ "SELECT product_info FROM products",+ )+ require.NoError(t, err)+ require.Equal(t, mysqlSpan.Results, starrocksSpan.Results)+ require.Equal(t, mysqlSpan.SourceColumns, starrocksSpan.SourceColumns)+ })++ t.Run("duplicate_database_case_uses_ignore_case_setting", func(t *testing.T) {+ gCtx := newOmniCaseCollisionTestQuerySpanContext()++ span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(context.Background(), "SELECT DB.t.a FROM DB.t")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "DB", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "DB", Table: "t"}}), span.SourceColumns)++ span, err = newOmniQuerySpanExtractor("db", gCtx, true).getOmniQuerySpan(context.Background(), "SELECT DB.t.a FROM DB.t")+ require.NoError(t, err)+ require.Equal(t, []base.QuerySpanResult{+ {Name: "a", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "t", Column: "a"}}), IsPlainField: true},+ }, span.Results)+ require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "DB", Table: "t"}}), span.SourceColumns)+ })+}++func sourceColumnSetFromResources(resources []base.ColumnResource) base.SourceColumnSet {+ result := make(base.SourceColumnSet)+ for _, resource := range resources {+ result[resource] = true+ }+ return result+}++func newOmniViewTestQuerySpanContext() base.GetQuerySpanContext {+ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "",+ Tables: []*storepb.TableMetadata{+ {+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ {Name: "b"},+ },+ },+ },+ Views: []*storepb.ViewMetadata{+ {+ Name: "v",+ Definition: "SELECT a AS va, b FROM t",+ },+ {+ Name: "v2",+ Definition: "SELECT va FROM v",+ },+ {+ Name: "broken_v",+ Definition: "SELECT missing FROM t",+ },+ },+ },+ },+ }+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ return base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ Engine: storepb.Engine_MYSQL,+ }+}++func newOmniTestQuerySpanContext() base.GetQuerySpanContext {+ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "",+ Tables: []*storepb.TableMetadata{+ {+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ {Name: "b"},+ {Name: "c"},+ },+ },+ {+ Name: "t1",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ {Name: "b"},+ {Name: "c"},+ },+ },+ {+ Name: "t2",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ {Name: "b"},+ {Name: "c"},+ },+ },+ {+ Name: "galleries",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id"},+ {Name: "city"},+ },+ },+ {+ Name: "paintings",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id"},+ {Name: "gallery_id"},+ },+ },+ {+ Name: "products",+ Columns: []*storepb.ColumnMetadata{+ {Name: "product_info"},+ },+ },+ {+ Name: "keyword_table",+ Columns: []*storepb.ColumnMetadata{+ {Name: "select"},+ {Name: "Camel"},+ },+ },+ },+ },+ },+ }+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ return base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ Engine: storepb.Engine_MYSQL,+ }+}++func newOmniStarRocksTestQuerySpanContext() base.GetQuerySpanContext {+ gCtx := newOmniTestQuerySpanContext()+ gCtx.Engine = storepb.Engine_STARROCKS+ return gCtx+}++func newOmniCaseCollisionTestQuerySpanContext() base.GetQuerySpanContext {+ dbMetadata := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "",+ Tables: []*storepb.TableMetadata{+ {+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ },+ },+ },+ },+ },+ }+ upperMetadata := &storepb.DatabaseSchemaMetadata{+ Name: "DB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "",+ Tables: []*storepb.TableMetadata{+ {+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "a"},+ },+ },+ },+ },+ },+ }+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{dbMetadata, upperMetadata})+ return base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ Engine: storepb.Engine_MYSQL,+ }+}diff --git a/backend/plugin/parser/mysql/query_span_omni_parity_test.go b/backend/plugin/parser/mysql/query_span_omni_parity_test.gonew file mode 100644index 0000000000..4add01e968--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_omni_parity_test.go@@ -0,0 +1,130 @@+package mysql++import (+ "context"+ "fmt"+ "io"+ "os"+ "reflect"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// TestMySQLOmniQuerySpanGoldenHarness is the strict cutover guard. It compares+// the package-internal omni path against the existing YAML corpus instead of+// comparing GetQuerySpan to itself after the production cutover.+func TestMySQLOmniQuerySpanGoldenHarness(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description,omitempty"`+ Statement string `yaml:"statement,omitempty"`+ DefaultDatabase string `yaml:"defaultDatabase,omitempty"`+ IgnoreCaseSensitve bool `yaml:"ignoreCaseSensitive,omitempty"`+ Metadata string `yaml:"metadata,omitempty"`+ QuerySpan *base.YamlQuerySpan `yaml:"querySpan,omitempty"`+ }++ type diff struct {+ fixture string+ index int+ description string+ statement string+ details string+ }++ var diffs []diff+ total := 0+ for _, testDataPath := range mysqlOmniProbeFixturePaths {+ engine := storepb.Engine_MYSQL+ if strings.Contains(testDataPath, "starrocks") {+ engine = storepb.Engine_STARROCKS+ }++ yamlFile, err := os.Open(testDataPath)+ if err != nil {+ t.Fatalf("open %s: %v", testDataPath, err)+ }+ byteValue, err := io.ReadAll(yamlFile)+ if closeErr := yamlFile.Close(); closeErr != nil && err == nil {+ err = closeErr+ }+ if err != nil {+ t.Fatalf("read %s: %v", testDataPath, err)+ }++ var testCases []testCase+ if err := yaml.Unmarshal(byteValue, &testCases); err != nil {+ t.Fatalf("yaml %s: %v", testDataPath, err)+ }++ for i, tc := range testCases {+ if strings.TrimSpace(tc.Statement) == "" {+ continue+ }+ total++++ omni, err := runMySQLOmniGoldenCase(context.Background(), tc.Statement, tc.Metadata, tc.DefaultDatabase, engine, tc.IgnoreCaseSensitve)+ if err != nil {+ diffs = append(diffs, diff{+ fixture: testDataPath,+ index: i,+ description: tc.Description,+ statement: tc.Statement,+ details: err.Error(),+ })+ continue+ }+ if !reflect.DeepEqual(tc.QuerySpan, omni) {+ diffs = append(diffs, diff{+ fixture: testDataPath,+ index: i,+ description: tc.Description,+ statement: tc.Statement,+ details: fmt.Sprintf("want=%+v omni=%+v", tc.QuerySpan, omni),+ })+ }+ }+ }++ t.Logf("MySQL omni query-span golden: %d/%d matched, %d diffs", total-len(diffs), total, len(diffs))+ for i, d := range diffs {+ if i >= 10 {+ t.Logf("... %d more diffs omitted", len(diffs)-i)+ break+ }+ t.Logf("[%s case %d %q] %s\n SQL: %s", d.fixture, d.index, d.description, d.details, firstMySQLOmniProbeLine(d.statement))+ }+ require.Empty(t, diffs)+}++func runMySQLOmniGoldenCase(+ ctx context.Context,+ statement string,+ metadataText string,+ defaultDatabase string,+ engine storepb.Engine,+ ignoreCaseSensitive bool,+) (*base.YamlQuerySpan, error) {+ metadata := &storepb.DatabaseSchemaMetadata{}+ if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(metadataText), metadata); err != nil {+ return nil, err+ }+ databaseMetadataGetter, databaseNameLister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{metadata})+ gCtx := base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: databaseMetadataGetter,+ ListDatabaseNamesFunc: databaseNameLister,+ Engine: engine,+ }++ omni, err := newOmniQuerySpanExtractor(defaultDatabase, gCtx, ignoreCaseSensitive).getOmniQuerySpan(ctx, statement)+ if err != nil {+ return nil, err+ }+ return omni.ToYaml(), nil+}diff --git a/backend/plugin/parser/mysql/query_span_omni_probe_test.go b/backend/plugin/parser/mysql/query_span_omni_probe_test.gonew file mode 100644index 0000000000..1a2f201c44--- /dev/null+++ b/backend/plugin/parser/mysql/query_span_omni_probe_test.go@@ -0,0 +1,260 @@+package mysql++import (+ "errors"+ "io"+ "os"+ "slices"+ "strings"+ "testing"++ "github.com/bytebase/omni/mysql/ast"+ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"+)++// TestMySQLOmniQuerySpanMigrationProbe verifies that omni can parse and expose+// the AST shapes the MySQL query-span migration will rely on. It does not use+// or change the production query-span path.+func TestMySQLOmniQuerySpanMigrationProbe(t *testing.T) {+ t.Run("FixtureParseCoverage", probeMySQLOmniFixtureParseCoverage)+ t.Run("StructuralInvariants", probeMySQLOmniStructuralInvariants)+}++type mysqlOmniProbeFixtureEntry struct {+ Description string `yaml:"description"`+ Statement string `yaml:"statement"`+}++var mysqlOmniProbeFixturePaths = []string{+ "test-data/query-span/query_type.yaml",+ "test-data/query-span/standard.yaml",+ "test-data/query-span/case_insensitive.yaml",+ "test-data/query-span/starrocks.yaml",+}++func probeMySQLOmniFixtureParseCoverage(t *testing.T) {+ type failure struct {+ fixture string+ index int+ statement string+ err error+ }++ totals := map[string][2]int{} // fixture -> [total, passed]+ var failures []failure+ for _, path := range mysqlOmniProbeFixturePaths {+ f, err := os.Open(path)+ require.NoError(t, err)+ data, err := io.ReadAll(f)+ require.NoError(t, err)+ require.NoError(t, f.Close())++ var entries []mysqlOmniProbeFixtureEntry+ require.NoError(t, yaml.Unmarshal(data, &entries))++ total := 0+ passed := 0+ for i, entry := range entries {+ if strings.TrimSpace(entry.Statement) == "" {+ continue+ }+ total+++ list, err := ParseMySQLOmni(entry.Statement)+ if err != nil {+ failures = append(failures, failure{+ fixture: path,+ index: i,+ statement: entry.Statement,+ err: err,+ })+ continue+ }+ if list == nil || list.Len() == 0 {+ failures = append(failures, failure{+ fixture: path,+ index: i,+ statement: entry.Statement,+ err: errEmptyOmniStatement,+ })+ continue+ }+ passed+++ }+ totals[path] = [2]int{total, passed}+ }++ t.Log("=== MySQL omni parse coverage per fixture ===")+ var keys []string+ for k := range totals {+ keys = append(keys, k)+ }+ slices.Sort(keys)+ totalAll, passedAll := 0, 0+ for _, k := range keys {+ v := totals[k]+ totalAll += v[0]+ passedAll += v[1]+ t.Logf(" %s: %d/%d", k, v[1], v[0])+ }+ t.Logf(" TOTAL: %d/%d", passedAll, totalAll)++ if len(failures) > 0 {+ t.Logf("=== %d parse failures ===", len(failures))+ for _, f := range failures {+ t.Logf(" [%s case %d] err: %v\n SQL: %s", f.fixture, f.index, f.err, firstMySQLOmniProbeLine(f.statement))+ }+ t.Errorf("omni failed to parse %d/%d fixture statements", len(failures), totalAll)+ }+}++var errEmptyOmniStatement = errors.New("omni returned no statements")++func firstMySQLOmniProbeLine(s string) string {+ s = strings.TrimSpace(s)+ if i := strings.IndexByte(s, '\n'); i >= 0 {+ return s[:i] + " ..."+ }+ return s+}++func probeMySQLOmniStructuralInvariants(t *testing.T) {+ t.Run("select_targets_aliases_and_stars", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "SELECT a, b AS alias_b, db.t.c, t.* FROM db.t AS t")+ require.Len(t, sel.TargetList, 4)++ first := mustMySQLOmniTargetColumn(t, sel.TargetList[0])+ require.Equal(t, "a", first.Column)+ require.Empty(t, first.Table)+ require.False(t, first.Star)++ aliasTarget, ok := sel.TargetList[1].(*ast.ResTarget)+ require.Truef(t, ok, "second target is %T", sel.TargetList[1])+ require.Equal(t, "alias_b", aliasTarget.Name)+ aliasColumn, ok := aliasTarget.Val.(*ast.ColumnRef)+ require.Truef(t, ok, "alias target value is %T", aliasTarget.Val)+ require.Equal(t, "b", aliasColumn.Column)++ qualified := mustMySQLOmniTargetColumn(t, sel.TargetList[2])+ require.Equal(t, "db", qualified.Schema)+ require.Equal(t, "t", qualified.Table)+ require.Equal(t, "c", qualified.Column)++ tableStar := mustMySQLOmniTargetColumn(t, sel.TargetList[3])+ require.Equal(t, "t", tableStar.Table)+ require.True(t, tableStar.Star)+ })++ t.Run("join_on_and_using", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "SELECT * FROM t1 JOIN db.t2 AS t2 ON t1.a = t2.a JOIN t3 USING (id)")+ require.Len(t, sel.From, 1)+ require.Len(t, collectMySQLOmniNodes[*ast.TableRef](sel), 3)+ require.NotEmpty(t, collectMySQLOmniNodes[*ast.OnCondition](sel))+ using := collectMySQLOmniNodes[*ast.UsingCondition](sel)+ require.Len(t, using, 1)+ require.Equal(t, []string{"id"}, using[0].Columns)+ })++ t.Run("derived_table_alias_and_column_aliases", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "SELECT x.c1 FROM (SELECT a, b FROM t) AS x(c1, c2)")+ require.Len(t, sel.From, 1)+ subquery, ok := sel.From[0].(*ast.SubqueryExpr)+ require.Truef(t, ok, "from item is %T", sel.From[0])+ require.Equal(t, "x", subquery.Alias)+ require.Equal(t, []string{"c1", "c2"}, subquery.Columns)+ require.NotNil(t, subquery.Select)+ })++ t.Run("recursive_cte_with_set_operation", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "WITH RECURSIVE cte(a, b) AS (SELECT a, b FROM t UNION SELECT a, b FROM t2) SELECT * FROM cte")+ require.Len(t, sel.CTEs, 1)+ cte := sel.CTEs[0]+ require.Equal(t, "cte", cte.Name)+ require.True(t, cte.Recursive)+ require.Equal(t, []string{"a", "b"}, cte.Columns)+ require.NotNil(t, cte.Select)+ require.Equal(t, ast.SetOpUnion, cte.Select.SetOp)+ require.NotNil(t, cte.Select.Left)+ require.NotNil(t, cte.Select.Right)+ })++ t.Run("correlated_scalar_subquery", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "SELECT city, (SELECT COUNT(*) FROM paintings p WHERE g.id = p.gallery_id) AS total FROM galleries g")+ require.Len(t, sel.TargetList, 2)+ subqueries := collectMySQLOmniNodes[*ast.SubqueryExpr](sel)+ require.Len(t, subqueries, 1)+ require.NotNil(t, subqueries[0].Select)++ var sawOuterRef, sawInnerRef bool+ for _, col := range collectMySQLOmniNodes[*ast.ColumnRef](sel) {+ if col.Table == "g" && col.Column == "id" {+ sawOuterRef = true+ }+ if col.Table == "p" && col.Column == "gallery_id" {+ sawInnerRef = true+ }+ }+ require.True(t, sawOuterRef, "expected correlated outer column g.id")+ require.True(t, sawInnerRef, "expected inner column p.gallery_id")+ })++ t.Run("json_table_table_source", func(t *testing.T) {+ sel := mustMySQLOmniSelect(t, "SELECT jt.id FROM t, JSON_TABLE(t.doc, '$[*]' COLUMNS(id INT PATH '$.id')) AS jt")+ jsonTables := collectMySQLOmniNodes[*ast.JsonTableExpr](sel)+ require.Len(t, jsonTables, 1)+ require.Equal(t, "jt", jsonTables[0].Alias)+ require.Len(t, jsonTables[0].Columns, 1)+ require.Equal(t, "id", jsonTables[0].Columns[0].Name)+ })++ t.Run("query_type_statement_roots", func(t *testing.T) {+ require.IsType(t, &ast.ExplainStmt{}, mustMySQLOmniNode(t, "EXPLAIN ANALYZE SELECT * FROM t"))+ require.IsType(t, &ast.ShowStmt{}, mustMySQLOmniNode(t, "SHOW DATABASES"))+ require.IsType(t, &ast.SetStmt{}, mustMySQLOmniNode(t, "SET CHARSET DEFAULT"))+ require.IsType(t, &ast.CreateTableStmt{}, mustMySQLOmniNode(t, "CREATE TABLE t(a INT)"))+ require.IsType(t, &ast.InsertStmt{}, mustMySQLOmniNode(t, "INSERT INTO t VALUES(1)"))+ })+}++func mustMySQLOmniNode(t *testing.T, sql string) ast.Node {+ t.Helper()+ list, err := ParseMySQLOmni(sql)+ require.NoError(t, err)+ require.NotNil(t, list)+ require.Len(t, list.Items, 1)+ require.NotNil(t, list.Items[0])+ return list.Items[0]+}++func mustMySQLOmniSelect(t *testing.T, sql string) *ast.SelectStmt {+ t.Helper()+ sel, ok := mustMySQLOmniNode(t, sql).(*ast.SelectStmt)+ require.Truef(t, ok, "statement is not *ast.SelectStmt: %s", sql)+ return sel+}++func mustMySQLOmniTargetColumn(t *testing.T, target ast.ExprNode) *ast.ColumnRef {+ t.Helper()+ switch v := target.(type) {+ case *ast.ColumnRef:+ return v+ case *ast.ResTarget:+ col, ok := v.Val.(*ast.ColumnRef)+ require.Truef(t, ok, "target value is %T", v.Val)+ return col+ default:+ t.Fatalf("target is %T", target)+ return nil+ }+}++func collectMySQLOmniNodes[T ast.Node](root ast.Node) []T {+ var nodes []T+ ast.Inspect(root, func(n ast.Node) bool {+ if typed, ok := n.(T); ok {+ nodes = append(nodes, typed)+ }+ return true+ })+ return nodes+}diff --git a/backend/plugin/parser/mysql/query_span_test.go b/backend/plugin/parser/mysql/query_span_test.goindex ba2bd77891..5b96181794 100644--- a/backend/plugin/parser/mysql/query_span_test.go+++ b/backend/plugin/parser/mysql/query_span_test.go@@ -7,7 +7,6 @@ import ("strings""testing"- "github.com/antlr4-go/antlr/v4""github.com/pkg/errors""github.com/stretchr/testify/require""gopkg.in/yaml.v3"@@ -171,15 +170,19 @@ func TestExtractTableRefs(t *testing.T) {}for _, test := range tests {- parseResult, err := ParseMySQL(test.statement)+ parseResult, err := ParseMySQLOmni(test.statement)require.NoError(t, err, "failed to parse statement: %s", test.statement)- require.Len(t, parseResult, 1, "expected one parse result for statement: %s", test.statement)- require.NotNil(t, parseResult[0].Tree, "parse tree is nil for statement: %s", test.statement)-- tree, ok := parseResult[0].Tree.(antlr.ParserRuleContext)- require.True(t, ok, "expected parse tree to be of type antlr.RuleContext for statement: %s", test.statement)-- resources := extractTableRefs("db", tree)- require.Equal(t, test.expected, resources, test.statement)+ require.Len(t, parseResult.Items, 1, "expected one parse result for statement: %s", test.statement)++ sourceColumns := collectOmniAccessTables(parseResult.Items[0], "db", false)+ var resources []base.SchemaResource+ for resource := range sourceColumns {+ resources = append(resources, base.SchemaResource{+ Database: resource.Database,+ Schema: resource.Schema,+ Table: resource.Table,+ })+ }+ require.ElementsMatch(t, test.expected, resources, test.statement)}}diff --git a/backend/plugin/parser/mysql/query_test.go b/backend/plugin/parser/mysql/query_test.goindex 7cbcc898ae..d8ffc40e47 100644--- a/backend/plugin/parser/mysql/query_test.go+++ b/backend/plugin/parser/mysql/query_test.go@@ -4,6 +4,8 @@ import ("testing""github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base")func TestValidateSQLForEditor(t *testing.T) {@@ -118,16 +120,23 @@ func TestValidateSQLForEditor(t *testing.T) {valid: true,gotAllQuery: false,},+ {+ statement: "SELECT * FORM invalid_table;",+ err: true,+ },}for _, test := range tests {- gotValid, gotAllQuery, err := validateQuery(test.statement)- if test.err {- require.Error(t, err)- } else {- require.NoError(t, err)- require.Equal(t, test.valid, gotValid, test.statement)- require.Equal(t, test.gotAllQuery, gotAllQuery, test.statement)- }+ t.Run(test.statement, func(t *testing.T) {+ gotValid, gotAllQuery, err := validateQuery(test.statement)+ if test.err {+ require.Error(t, err)+ require.ErrorAs(t, err, new(*base.SyntaxError))+ } else {+ require.NoError(t, err)+ require.Equal(t, test.valid, gotValid, test.statement)+ require.Equal(t, test.gotAllQuery, gotAllQuery, test.statement)+ }+ })}}diff --git a/backend/plugin/parser/mysql/query_type.go b/backend/plugin/parser/mysql/query_type.godeleted file mode 100644index 5b0b86e55c..0000000000--- a/backend/plugin/parser/mysql/query_type.go+++ /dev/null@@ -1,98 +0,0 @@-package mysql--import (- "github.com/antlr4-go/antlr/v4"- "github.com/bytebase/parser/mysql"-- "github.com/bytebase/bytebase/backend/plugin/parser/base"-)--func isTopLevel(ctx antlr.Tree) bool {- if ctx == nil {- return true- }- switch ctx := ctx.(type) {- case *mysql.SimpleStatementContext:- return isTopLevel(ctx.GetParent())- case *mysql.QueryContext, *mysql.ScriptContext:- return true- default:- return false- }-}--type queryTypeListener struct {- *mysql.BaseMySQLParserListener-- allSystems bool- result base.QueryType- isExplainAnalyze bool-}--func (l *queryTypeListener) EnterSimpleStatement(ctx *mysql.SimpleStatementContext) {- if !isTopLevel(ctx) {- return- }-- switch {- case ctx.AlterStatement() != nil,- ctx.CreateStatement() != nil,- ctx.DropStatement() != nil,- ctx.RenameTableStatement() != nil,- ctx.TruncateTableStatement() != nil,- ctx.ImportStatement() != nil:- l.result = base.DDL- case ctx.CallStatement() != nil,- ctx.DeleteStatement() != nil,- ctx.DoStatement() != nil,- ctx.HandlerStatement() != nil,- ctx.InsertStatement() != nil,- ctx.LoadStatement() != nil,- ctx.ReplaceStatement() != nil,- ctx.UpdateStatement() != nil,- ctx.TransactionOrLockingStatement() != nil,- ctx.ReplicationStatement() != nil,- ctx.PreparedStatement() != nil:- l.result = base.DML- case ctx.SetStatement() != nil:- if len(ctx.SetStatement().StartOptionValueList().AllPASSWORD_SYMBOL()) == 0 {- l.result = base.Select- }- case ctx.ShowStatement() != nil:- l.result = base.SelectInfoSchema- case ctx.UtilityStatement() != nil:- if ctx.UtilityStatement().DescribeStatement() != nil {- l.result = base.SelectInfoSchema- }- if ctx.UtilityStatement().ExplainStatement() != nil {- if ctx.UtilityStatement().ExplainStatement().ANALYZE_SYMBOL() != nil {- l.isExplainAnalyze = true- explainableStatement := ctx.UtilityStatement().ExplainStatement().ExplainableStatement()- switch {- case explainableStatement.SelectStatement() != nil:- l.result = base.Select- case explainableStatement.DeleteStatement() != nil, explainableStatement.InsertStatement() != nil, explainableStatement.ReplaceStatement() != nil, explainableStatement.UpdateStatement() != nil:- l.result = base.DML- default:- l.result = base.Explain- }- } else {- l.result = base.Explain- }- }- default:- l.result = base.Select- }-}--func (l *queryTypeListener) EnterSelectStatement(ctx *mysql.SelectStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }-- if l.allSystems {- l.result = base.SelectInfoSchema- } else {- l.result = base.Select- }-}diff --git a/docs/plans/2026-04-27-mysql-query-span-omni-migration.md b/docs/plans/2026-04-27-mysql-query-span-omni-migration.mdnew file mode 100644index 0000000000..505d15ca4f--- /dev/null+++ b/docs/plans/2026-04-27-mysql-query-span-omni-migration.md@@ -0,0 +1,518 @@+# MySQL Query Span Omni Migration Implementation Plan++> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.++**Goal:** Migrate MySQL query-span extraction from ANTLR parse-tree walking to omni MySQL AST without runtime fallback.++**Architecture:** Follow the MSSQL migration model: build a package-internal omni extractor that mirrors the existing extractor's behavior, prove it with probe tests, focused behavior tests, and a full-corpus golden harness, then cut over once fixture output is unchanged. Do not use the PostgreSQL catalog-analyzer model for this migration; MySQL already has a hand-written extractor whose behavior can be translated directly onto omni AST.++**Tech Stack:** Go, `github.com/bytebase/omni/mysql/ast`, `backend/plugin/parser/base` query-span model, existing YAML fixtures under `backend/plugin/parser/mysql/test-data/query-span/`.++## Current Status++- Probe coverage exists in `backend/plugin/parser/mysql/query_span_omni_probe_test.go`.+- Omni parses 30/30 current MySQL query-span fixture statements.+- Probe assertions cover aliases, stars, joins, derived tables, CTEs, set operations, correlated subqueries, `JSON_TABLE`, and query-type root nodes.+- `backend/plugin/parser/mysql/query_span_extractor_omni.go` is the production MySQL query-span extractor.+- Strict golden coverage is 30/30 matched and 0 diffs across the current MySQL query-span fixture corpus.+- Post-review systematic regression tests cover derived-table column alias lists, `IN (SELECT ...)` result lineage, explicit long-tail expression nodes, legacy DML roots (`CALL`, `DO`, `HANDLER`), and `TABLE` / `VALUES` select-family roots.+- The omni expression extractor now fails on unsupported expression node types instead of silently returning empty lineage.+- Public `GetQuerySpan` is omni-backed through `querySpanExtractor.getQuerySpan`.+- `query_span*.go` production code no longer depends on `ParseMySQL`, `GetANTLRAST`, `antlr4-go`, or `github.com/bytebase/parser/mysql`.++## Non-Negotiables++- No runtime feature flag.+- No environment switch.+- No fallback from omni to ANTLR after cutover.+- No partial production rollout.+- The ANTLR path may exist only as a development reference before cutover.+- The cutover happens only after the omni path has zero parity diffs across the current fixture corpus.+- Existing YAML fixture expectations remain unchanged at cutover.++## Final Shape++The final MySQL query-span implementation should look like this:++```text+GetQuerySpan+ -> newQuerySpanExtractor(...)+ -> q.getQuerySpan(ctx, stmt.Text)+ -> ParseMySQLOmni(stmt)+ -> collectOmniAccessTables(root)+ -> isMixedQuery(...)+ -> classifyOmniQueryType(root, allSystems)+ -> if non-SELECT: return type + access tables+ -> extractFromSelectRoot(*ast.SelectStmt | *ast.TableStmt | *ast.ValuesStmt)+ -> processCTEs+ -> extractFromSetOp+ -> extractFromClause+ -> collect predicate/source tables as legacy behavior requires+ -> extractTargetList+ -> return QuerySpan+```++Final file ownership:++- `backend/plugin/parser/mysql/query_span.go`+ - Stays as the public registration and `GetQuerySpan` wrapper.+ - Calls the sole omni-backed `querySpanExtractor.getQuerySpan`.++- `backend/plugin/parser/mysql/query_span_extractor.go`+ - Final home for shared extractor state and string/metadata helpers.+ - Keeps helpers that do not depend on ANTLR contexts:+ - `getAllTableColumnSources`+ - `getFieldColumnSource`+ - `filterClusterName`+ - `findTableSchema`+ - `getColumnsForView`+ - `isMixedQuery`+ - `isSystemResource`+ - Deletes ANTLR listener and ANTLR context methods after cutover.++- `backend/plugin/parser/mysql/query_span_extractor_omni.go`+ - Final home for all omni AST extraction logic.+ - May be split if it grows too large:+ - `query_span_omni_expr.go`+ - `query_span_omni_from.go`+ - `query_span_omni_access.go`+ - Starts package-internal while ANTLR remains reference.+ - Becomes the production path at cutover.++- `backend/plugin/parser/mysql/query_type.go`+ - Current ANTLR `queryTypeListener` is deleted or replaced by `classifyOmniQueryType`.++- `backend/plugin/parser/mysql/omni.go`+ - `AsANTLRAST()` fallback is removed only after no MySQL caller needs it.+ - This migration removes query-span's dependency; other MySQL modules must be checked separately before deleting fallback globally.++- Tests:+ - `query_span_omni_probe_test.go` stays through cutover, then can remain as a parser regression test.+ - `query_span_extractor_omni_test.go` stays as focused unit coverage.+ - `query_span_omni_parity_test.go` remains as a golden harness after cutover. It must compare the internal omni extractor directly against YAML expectations, not `GetQuerySpan` against the internal omni path.++## Coverage Matrix++The current fixture corpus is the minimum gate. Add focused tests for each bucket before implementing that bucket.++| Bucket | Required behavior |+|---|---|+| Query type | `SELECT`, `TABLE`, `VALUES`, `EXPLAIN`, `EXPLAIN ANALYZE SELECT`, `SHOW`, `SET`, DDL, DML, all-system SELECT, legacy DML roots (`CALL`, `DO`, `HANDLER`) |+| Access tables | top-level FROM, joined tables, derived-table body tables, CTE body tables, subqueries in SELECT/WHERE, system-table suppression |+| Target list | constants, bare columns, qualified columns, aliases, expression names, `*`, `table.*` |+| Expressions | column refs, literals, binary/unary ops, functions, aggregates, `CASE`, `CAST`, `BETWEEN`, `IN` value list, `IN` subquery, `LIKE`, `IS`, `EXISTS`, scalar subquery, `CONVERT`, `COLLATE`, `MATCH`, `ROW`, `MEMBER OF`, `INTERVAL`, window args |+| FROM | `TableRef`, aliases, comma joins, `JoinClause`, `ON`, `USING`, nested joins |+| Derived tables | subquery table source, derived alias, derived column alias list, nested derived tables |+| Set operations | `UNION`, `UNION ALL`, recursive CTE union, positional source-column merge |+| CTE | non-recursive CTE, nested CTE, explicit CTE column list, recursive CTE anchor/recursive merge |+| JSON_TABLE | JSON table columns, `priorTableInFrom`, source lineage from JSON expression owner |+| Views | `getColumnsForView` recurses through omni extractor and applies view output columns |+| Case sensitivity | existing `ignoreCaseSensitive` behavior preserved |+| Not found | missing database/table/column errors map to `QuerySpan.NotFoundError` as today |+| StarRocks | current two StarRocks fixtures keep matching |++## ANTLR To Omni Mapping++| Current ANTLR item | Final omni item |+|---|---|+| `getQuerySpan` | `getOmniQuerySpan` renamed back to `getQuerySpan` at cutover |+| `queryTypeListener` | `classifyOmniQueryType(ast.Node, allSystems)` |+| `selectOnlyListener` | direct root dispatch in `getOmniQuerySpan` |+| `extractContext` | direct `extractFromSelectStmt` / statement type switch |+| `extractSelectStatement` | `extractFromSelectStmt(*ast.SelectStmt)` |+| `extractQueryExpression` | `extractFromSelectStmt` including CTE + set-op handling |+| `extractQueryExpressionParens` | unnecessary; omni AST is already normalized |+| `extractQueryExpressionBody` | `extractFromSetOp` or simple select body handling |+| `extractQueryPrimary` | `extractFromSelectStmt`, `extractTableStmt`, `extractValuesStmt` as needed |+| `extractExplicitTable` | `resolveTableRef` / `extractTableStmt` |+| `extractTableValueConstructor` | `extractValuesStmt` if fixtures require it |+| `extractQuerySpecification` | `extractFromSelectStmt` simple-select branch |+| `extractSelectItemList` | `extractTargetList([]ast.ExprNode, fromSources)` |+| `extractSelectItem` | `extractTarget` |+| `extractSourceColumnSetFromExpr` | `resolveExpression(ast.ExprNode)` |+| `extractSourceColumnSetFromExprList` | `mergeExpressionSources(...ast.ExprNode)` |+| `extractTableWild` | `expandStar(*ast.ColumnRef)` |+| `extractTableSourcesFromFromClause` | `extractFromClause([]ast.TableExpr)` |+| `extractTableReferenceList` | `extractFromClause` |+| `extractTableReference` | `extractTableSource(ast.TableExpr)` |+| `extractJoinedTable` | `extractJoin(*ast.JoinClause)` |+| `extractTableFactor` | `extractTableSource` type switch |+| `extractTableFunction` | `extractJsonTable(*ast.JsonTableExpr)` |+| `extractTableReferenceListParens` | unnecessary unless omni exposes nested table lists |+| `extractSubquery` | `extractSubqueryAsPseudo(*ast.SelectStmt)` |+| `extractDerivedTable` | `extractDerivedTable(*ast.SubqueryExpr)` |+| `extractSingleTable` | `resolveTableRef(*ast.TableRef)` |+| `extractSingleTableParens` | unnecessary |+| `extractCommonTableExpression` | `processCTEs([]*ast.CommonTableExpr)` |+| `extractRecursiveCTE` | `extractRecursiveCTE(*ast.CommonTableExpr)` |+| `extractNonRecursiveCTE` | `extractNonRecursiveCTE(*ast.CommonTableExpr)` |+| `recursiveCTEExtractListener` | direct recursive CTE anchor/recursive branch extraction |+| `getAccessTables` / `accessTableListener` | `collectOmniAccessTables(ast.Node, defaultDatabase)` |+| `extractTableRefs` / `resourceExtractListener` | explicit table-source traversal for access tables |++## Target API++Final extractor methods:++```go+func (q *querySpanExtractor) getQuerySpan(ctx context.Context, stmt string) (*base.QuerySpan, error)+func (q *querySpanExtractor) extractFromSelectStmt(sel *ast.SelectStmt) (*base.PseudoTable, error)+func (q *querySpanExtractor) extractFromSetOp(sel *ast.SelectStmt) (*base.PseudoTable, error)+func (q *querySpanExtractor) processCTEs(ctes []*ast.CommonTableExpr) error+func (q *querySpanExtractor) extractTableSource(expr ast.TableExpr) ([]base.TableSource, error)+func (q *querySpanExtractor) resolveTableRef(ref *ast.TableRef) (base.TableSource, error)+func (q *querySpanExtractor) resolveExpression(expr ast.ExprNode) (base.QuerySpanResult, error)+func (q *querySpanExtractor) cloneForSubquery() *querySpanExtractor+func collectOmniAccessTables(root ast.Node, defaultDatabase string) base.SourceColumnSet+```++During migration, these may live on `*omniQuerySpanExtractor`. At cutover, merge or rename so there is only one production extractor type.++## Execution Plan++Every implementation task follows TDD:++1. Write the focused failing test.+2. Run the focused test and confirm it fails for the expected missing behavior.+3. Implement the smallest behavior.+4. Run the focused test.+5. Run golden harness and record matched/diff count.+6. Run existing `TestGetQuerySpan` to protect the reference path before cutover.++### Phase 0: Probe And Scaffold++Status: done.++Files:++- `backend/plugin/parser/mysql/query_span_omni_probe_test.go`+- `backend/plugin/parser/mysql/query_span_extractor_omni.go`+- `backend/plugin/parser/mysql/query_span_extractor_omni_test.go`+- `backend/plugin/parser/mysql/query_span_omni_parity_test.go`++Proof:++```bash+go test -v -count=1 github.com/bytebase/bytebase/backend/plugin/parser/mysql -run '^(TestMySQLOmniQuerySpanMigrationProbe|TestOmniQuerySpanScaffold_QueryTypesAndAccessTables|TestMySQLOmniQuerySpanGoldenHarness|TestGetQuerySpan)$'+```++### Phase 1: Simple SELECT Result Columns++Status: done.++Goal: Make basic target-list fixtures match.++Tests first:++- `SELECT 1`+- `SELECT a FROM t`+- `SELECT a AS x FROM t`+- `SELECT a, t.b, db.t.c FROM t`+- `SELECT * FROM t`+- `SELECT *, a FROM t`++Implementation:++- Add `extractFromSelectStmt` simple branch.+- Add `extractTargetList`.+- Add `resolveExpression` for:+ - `*ast.ColumnRef`+ - literals+ - `*ast.ResTarget`+- Add `expandStar`.++Expected parity movement:++- Several `standard.yaml` simple SELECT cases should move from diff to match.++### Phase 2: Expression Source Merging++Status: done.++Goal: Match expression lineage and names for non-subquery expressions.++Tests first:++- arithmetic: `a-b AS c1`+- comparison: `a=b AS c2`+- function: `MAX(a)`+- nested function args+- `CASE`+- `CAST`+- `BETWEEN`+- `IN` value list+- `LIKE`+- `IS NULL`+- window function args if omni fixtures or added tests require them++Implementation:++- Extend `resolveExpression`.+- Add `mergeExpressionSources`.+- Preserve `IsPlainField` semantics:+ - plain column ref: true+ - literal constant: true, matching current MySQL behavior for `SELECT 1`+ - expression over columns/functions/subqueries: false++### Phase 3: FROM, JOIN, Alias, And Scope++Status: done.++Goal: Resolve table aliases and joined table sources.++Tests first:++- `FROM t AS x`+- `x.a`+- `JOIN ... ON`+- `JOIN ... USING(a)`+- comma join+- nested join+- alias shadowing between table aliases and physical names++Implementation:++- Add `extractFromClause`.+- Add `extractTableSource`.+- Add `extractJoin`.+- Add alias-aware table source wrappers if current `base.TableSource` APIs require pseudo renaming.+- Ensure `tableSourceFrom` ordering matches current ANTLR behavior.++### Phase 4: Derived Tables And Subqueries++Status: done.++Goal: Match derived table and scalar/correlated subquery behavior.++Tests first:++- `(SELECT a,b FROM t) AS x`+- `(SELECT a,b FROM t) AS x(c1,c2)`+- scalar subquery constant+- scalar subquery from table+- correlated scalar subquery+- `WHERE a IN (SELECT a FROM t)`+- `EXISTS (SELECT 1 FROM t WHERE ...)`++Implementation:++- Add `cloneForSubquery`.+- Add `extractDerivedTable`.+- Add `extractSubqueryAsPseudo`.+- Merge subquery source columns into the correct result or access-table set according to current parity.+- Fix `collectOmniAccessTables` to include subqueries in target lists and predicates, not only FROM.++### Phase 5: Set Operations++Status: done.++Goal: Match positional lineage for `UNION` and recursive CTE set-op bodies.++Tests first:++- `SELECT a FROM t UNION SELECT b FROM t2`+- `UNION ALL`+- column count mismatch behavior if current code errors+- set-op inside derived table++Implementation:++- Add `extractFromSetOp`.+- Merge result source columns positionally.+- Preserve anchor-side result names.++### Phase 6: CTEs++Status: done.++Goal: Match all existing CTE fixtures, including recursive cases.++Tests first:++- simple CTE+- nested CTE+- explicit CTE column aliases+- recursive CTE with explicit aliases+- recursive CTE without aliases++Implementation:++- Add `processCTEs`.+- Add `extractNonRecursiveCTE`.+- Add `extractRecursiveCTE`.+- Preserve current shadowing behavior: nearest CTE wins over physical table when database is unspecified.++### Phase 7: JSON_TABLE++Status: done.++Goal: Match JSON table fixture behavior.++Tests first:++- current `JSON_TABLE` fixture.+- JSON table columns derive source from owner JSON expression.+- `priorTableInFrom` resolves `t.doc` inside `JSON_TABLE(t.doc, ...)`.++Implementation:++- Add `extractJsonTable`.+- Add JSON table column pseudo results.+- Thread `priorTableInFrom` exactly as current extractor does.++### Phase 8: Views And Metadata Recursion++Status: done.++Goal: Keep view-derived columns working with omni recursion.++Tests first:++- existing StarRocks view fixtures.+- MySQL view metadata with `SELECT *`.+- view with aliases.++Implementation:++- Change `getColumnsForView` to call omni extractor once the needed phases are ready.+- Keep old behavior until cutover if needed for reference path.++### Phase 9: NotFound And Mixed System/User Behavior++Status: done.++Goal: Preserve error recovery and masking-relevant behavior.++Tests first:++- missing table+- missing column+- mixed `mysql.user` and user table query+- all-system table query returns `SelectInfoSchema` and empty source columns++Implementation:++- Route `ResourceNotFoundError` into `QuerySpan.NotFoundError`.+- Preserve `MixUserSystemTablesError`.+- Ensure all-system access tables are suppressed in returned span.++### Phase 10: Strict Golden Gate++Status: done.++Goal: Make the fixture harness strict and reach zero diffs against checked-in YAML expectations.++Steps:++1. Change the harness to fail when diffs exist.+2. Run it against all current fixtures.+3. Fix remaining diffs.+4. After cutover, keep the harness as a golden check and ensure it does not compare `GetQuerySpan` to the same internal omni path.++Proof:++```bash+go test -v -count=1 github.com/bytebase/bytebase/backend/plugin/parser/mysql -run '^TestMySQLOmniQuerySpanGoldenHarness$'+```++Current result: `30/30 matched, 0 diffs`.++### Phase 10.5: Post-Review Systematic Correction++Status: done.++Goal: Close the review findings and add regression coverage for the migration failure modes that let them through.++Root causes:++1. The migration moved from ANTLR's recursive parse-tree traversal to omni's explicit AST type switches, but some omni child fields were not wired into lineage extraction.+2. Probe tests verified AST shape availability, not extractor behavior.+3. The post-cutover golden harness initially used `GetQuerySpan` as the reference even though `GetQuerySpan` already delegated to omni.+4. The fixture corpus was too small to cover long-tail root statements and expression nodes from the old extractor.++Regression tests:++- `TestOmniQuerySpanSystematicMigrationRegressions/derived_table_column_aliases_are_applied`+- `TestOmniQuerySpanSystematicMigrationRegressions/in_subquery_sources_are_part_of_result_lineage`+- `TestOmniQuerySpanSystematicMigrationRegressions/explicit_expression_nodes_do_not_drop_lineage`+- `TestOmniQuerySpanSystematicMigrationRegressions/legacy_dml_roots_stay_dml`+- `TestOmniQuerySpanSystematicMigrationRegressions/table_and_values_roots_return_select_results`+- `TestMySQLOmniQuerySpanGoldenHarness`++Implementation corrections:++- Apply `SubqueryExpr.Columns` to derived table pseudo columns with length validation.+- Merge `InExpr.Select` result sources into the expression lineage.+- Add explicit expression handlers for `EXISTS`, `CONVERT`, `COLLATE`, `MATCH`, `ROW`, `MEMBER OF`, `INTERVAL`, and `DEFAULT`.+- Return an error for unsupported omni expression node types instead of silently returning empty lineage.+- Classify `CALL`, `DO`, and `HANDLER` roots as DML.+- Extract `TABLE` and `VALUES` roots as select-family results.++### Phase 11: Cutover++Status: done.++Goal: Make omni the only MySQL query-span production path.++Steps:++1. Change public `GetQuerySpan` path to use the omni extractor.+2. Rename or merge `omniQuerySpanExtractor` into `querySpanExtractor`.+3. Delete ANTLR context methods from query-span code.+4. Delete `selectOnlyListener`, `accessTableListener`, `resourceExtractListener`, and `recursiveCTEExtractListener`.+5. Delete ANTLR `queryTypeListener` or leave only if non-query-span code still uses it.+6. Keep `ParseMySQL` only for other modules that still require it.+7. Run no-ANTLR query-span grep:++```bash+rg 'ParseMySQL\(|GetANTLRAST|antlr4-go|github.com/bytebase/parser/mysql' backend/plugin/parser/mysql/query_span*.go+```++Expected: no production query-span dependency. Test-only parity code may be deleted or excluded.++### Phase 12: Cleanup++Status: done.++Goal: Remove temporary migration scaffolding.++Steps:++1. Decide whether to keep `query_span_omni_probe_test.go` as a permanent parser-shape regression test.+2. Keep `query_span_omni_parity_test.go` as a golden harness after `TestGetQuerySpan` is omni-backed and strict fixture tests pass.+3. Update this plan with final status.+4. Run full required Go checks.++## Global Verification++Focused query-span proof:++```bash+go test -v -count=1 github.com/bytebase/bytebase/backend/plugin/parser/mysql -run '^(TestGetQuerySpan|TestMySQLOmniQuerySpanMigrationProbe|TestOmniQuerySpan|TestMySQLOmniQuerySpanGoldenHarness)'+```++Package proof:++```bash+go test -v -count=1 github.com/bytebase/bytebase/backend/plugin/parser/mysql+```++Repository gates before PR:++```bash+gofmt -w <modified go files>+golangci-lint run --allow-parallel-runners+go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go+```++## Cutover Gate++- Probe test passes.+- Focused omni extractor tests pass.+- Strict golden harness reports 0 diffs.+- Existing `TestGetQuerySpan` passes unchanged after switching to omni.+- No production query-span dependency on ANTLR remains.+- `golangci-lint run --allow-parallel-runners` passes.+- Server build passes.++Only after this gate is satisfied should the branch be merged.diff --git a/docs/plans/SCENARIOS-mysql-query-span-omni.md b/docs/plans/SCENARIOS-mysql-query-span-omni.mdnew file mode 100644index 0000000000..eaeee73e42--- /dev/null+++ b/docs/plans/SCENARIOS-mysql-query-span-omni.md@@ -0,0 +1,261 @@+# MySQL Query Span Omni Scenarios++> Goal: Prove the omni MySQL query-span extractor preserves legacy MySQL query-span behavior without runtime fallback.+> Verification: each scenario must have an automated assertion against legacy-compatible query-span output, query type, access-table set, or error semantics.+> Reference sources: legacy ANTLR MySQL query-span extractor, legacy MySQL query type listener, current YAML query-span fixtures, omni MySQL AST node definitions, MySQL query-span migration plan.++Status: [ ] pending, [x] passing, [~] partial.++---++## Phase 1: Statement Type And Root Selection++### 1.1 Select-Family Roots++- [x] Plain `SELECT` returns `Select` and extracts target-list results.+- [x] `TABLE t` returns `Select` and expands to the referenced table columns.+- [x] `VALUES ROW(...)` returns `Select` and derives result columns from the first row.+- [x] `VALUES ROW(DEFAULT)` returns a constant `DEFAULT` result with empty source columns.+- [x] `SELECT ... INTO` preserves the legacy unsupported/error behavior.+- [x] `EXPLAIN SELECT ...` returns `Explain` without result columns.+- [x] `EXPLAIN ANALYZE SELECT ...` returns `Select` and preserves select access tables.+- [x] `EXPLAIN ANALYZE TABLE t` returns `Select` and preserves table access.+- [x] `EXPLAIN ANALYZE VALUES ROW(...)` returns `Select` without result lineage extraction.+- [x] `EXPLAIN ANALYZE INSERT ...` returns `DML` and no result columns.+- [x] `DESCRIBE ...` returns `SelectInfoSchema` like legacy metadata reads.+- [x] `DESC` followed by non-space whitespace returns `SelectInfoSchema`.+- [x] Multiple statements in one query return the legacy single-statement error.++### 1.2 Query Type Buckets++- [x] `SHOW ...` returns `SelectInfoSchema`.+- [x] non-password `SET ...` returns `Select`.+- [x] password-changing `SET PASSWORD ...` preserves legacy query type behavior.+- [x] `CREATE TABLE ...` returns `DDL`.+- [x] `CREATE DATABASE ...` returns `DDL`.+- [x] `CREATE VIEW ...` returns `DDL`.+- [x] user/role administration statements such as `CREATE USER`, `ALTER USER`, `DROP USER`, `GRANT`, and `REVOKE` return non-select query types.+- [x] `ALTER TABLE ...` returns `DDL`.+- [x] `DROP TABLE ...` returns `DDL`.+- [x] `RENAME TABLE ...` returns `DDL`.+- [x] `TRUNCATE TABLE ...` returns `DDL`.+- [x] `IMPORT TABLE ...` returns `DDL`.+- [x] `INSERT ...` returns `DML`.+- [x] `REPLACE ...` returns `DML`.+- [x] `UPDATE ...` returns `DML`.+- [x] `DELETE ...` returns `DML`.+- [x] `LOAD DATA ...` returns `DML`.+- [x] `CALL ...` returns `DML`.+- [x] `DO ...` returns `DML`.+- [x] `HANDLER ... OPEN` returns `DML`.+- [x] `HANDLER ... READ` returns `DML`.+- [x] `HANDLER ... CLOSE` returns `DML`.+- [x] transaction and locking statements return `DML`.+- [x] replication statements return `DML`.+- [x] prepared statements return `DML`.+- [x] unhandled omni statement roots fail closed instead of falling through to `Select`.++## Phase 2: Target List And Expression Lineage++### 2.1 Target List Basics++- [x] Literal target `SELECT 1` returns a literal result with empty source columns.+- [x] Bare column target returns that column as a plain field.+- [x] qualified column target resolves table and database qualifiers.+- [x] target alias overrides expression-derived result name.+- [x] unaliased expression uses the legacy expression text as result name.+- [x] `SELECT * FROM t` expands all source columns in table order.+- [x] `SELECT t.* FROM t` expands only the qualified table columns.+- [x] `SELECT *, a FROM t` preserves star expansion plus explicit target.+- [x] duplicate output names preserve legacy ordering and duplicate result entries.+- [x] reserved-word quoted identifiers resolve like legacy identifiers.++### 2.2 Expression Node Coverage++- [x] binary arithmetic expression merges both operand source columns.+- [x] binary comparison expression merges both operand source columns.+- [x] JSON extraction binary operators merge object and path expression sources.+- [x] unary expression preserves operand source columns and is not a plain field.+- [x] function call merges argument source columns.+- [x] aggregate function merges argument source columns.+- [x] window function merges argument, partition, and order source columns.+- [x] function separator expression contributes lineage where applicable.+- [x] function-level `ORDER BY` expressions contribute lineage where applicable.+- [x] scalar subquery result merges selected result source columns.+- [x] `EXISTS (SELECT ...)` returns subquery result lineage and is not a plain field.+- [x] `CASE` expression merges operand, condition, result, and default sources.+- [x] `BETWEEN` expression merges expression, lower, and upper sources.+- [x] `IN` value-list expression merges left expression and list item sources.+- [x] `IN (SELECT ...)` merges left expression and subquery result sources.+- [x] `LIKE` expression merges expression, pattern, and escape sources.+- [x] `IS NULL` and related `IS` tests preserve expression sources.+- [x] `CAST` expression preserves inner expression lineage.+- [x] `EXTRACT` expression preserves inner expression lineage.+- [x] `INTERVAL` expression preserves value expression lineage.+- [x] `COLLATE` expression preserves inner expression lineage.+- [x] `MATCH ... AGAINST ...` merges matched columns and search expression sources.+- [x] `CONVERT(expr USING charset)` preserves inner expression lineage.+- [x] `DEFAULT` expression returns empty source columns.+- [x] `ROW(...)` expression merges item sources.+- [x] `MEMBER OF` expression merges value and JSON-array sources.+- [x] user variable reference preserves legacy empty-lineage behavior.+- [x] system variable reference preserves legacy empty-lineage behavior.+- [ ] unknown future expression nodes fail closed or get explicit coverage before release.++## Phase 3: FROM And Table Sources++### 3.1 Tables, Aliases, And Stars++- [x] single table source resolves physical table columns.+- [x] table alias changes the visible table name for column resolution.+- [x] database-qualified table source resolves against the qualified database.+- [x] cluster-qualified StarRocks table source filters cluster name as before.+- [x] parenthesized single table source resolves like the unparenthesized table.+- [x] comma-separated table sources expose all tables to target resolution.+- [x] `DUAL` produces no table source and allows literal targets.+- [x] `DUAL` join operands remain non-nil empty table sources and do not panic.+- [x] missing table maps to `NotFoundError` fail-open behavior.+- [x] missing column maps to `NotFoundError` fail-open behavior.++### 3.2 Joins++- [x] inner join exposes left and right table columns.+- [x] cross join exposes left and right table columns.+- [x] straight join exposes left and right table columns.+- [x] left outer join exposes left and right table columns.+- [x] right outer join exposes left and right table columns.+- [x] `JOIN ... USING(col)` merges duplicate `USING` columns.+- [x] natural join merges common columns.+- [x] natural left join merges common columns.+- [x] natural right join merges common columns.+- [x] `JOIN ... ON` expression contributes access-table dependencies.+- [x] nested join tree preserves visible source ordering.+- [x] parenthesized table reference list behaves like legacy cross-join expansion.++### 3.3 Derived Tables And Table Functions++- [x] derived table body contributes access-table dependencies.+- [x] derived table alias controls visible table qualifier.+- [x] derived table column alias list renames output columns.+- [x] derived table column alias count mismatch returns legacy error.+- [x] nested derived tables preserve lineage through each layer.+- [x] lateral derived table preserves correlation behavior.+- [x] `JSON_TABLE` exposes declared columns as pseudo-table columns.+- [x] `JSON_TABLE` column lineage comes from the JSON document expression.+- [x] nested `JSON_TABLE` columns flatten in legacy order.+- [x] `JSON_TABLE` document expressions contribute access-table dependencies.+- [x] `JSON_TABLE` with alias omitted is rejected because MySQL requires a table-function alias.++## Phase 4: Subqueries, CTEs, And Set Operations++### 4.1 Subquery Scope++- [x] scalar subquery in target list contributes selected source columns.+- [x] correlated scalar subquery can resolve outer query columns.+- [x] subquery in `WHERE` contributes access-table dependencies.+- [x] subquery in `HAVING` contributes access-table dependencies.+- [x] subquery in `ORDER BY` contributes access-table dependencies.+- [x] nested subqueries preserve nearest-scope alias shadowing.+- [x] unqualified inner subquery columns follow legacy resolution order.+- [x] outer table alias shadowed by inner alias follows legacy resolution.++### 4.2 Common Table Expressions++- [x] non-recursive CTE exposes selected columns.+- [x] non-recursive CTE explicit column list renames output columns.+- [x] non-recursive CTE column count mismatch returns legacy error.+- [x] nested CTE references resolve in legacy visibility order.+- [x] recursive CTE merges anchor and recursive branch source columns.+- [x] recursive CTE explicit column list count mismatch returns legacy error.+- [x] recursive CTE reaches a stable source-column closure.+- [x] left-deep recursive CTE set operations register the recursive CTE before resolving recursive left branches.+- [x] CTE name shadows physical table name according to legacy behavior.+- [x] later CTE cannot be referenced by earlier CTE unless legacy allowed it.++### 4.3 Set Operations++- [x] `UNION` merges left and right source columns by position.+- [x] `UNION ALL` merges left and right source columns by position.+- [x] `INTERSECT` preserves legacy support or error behavior.+- [x] `EXCEPT` preserves legacy support or error behavior.+- [x] set operation result names come from the left side.+- [x] set operation column count mismatch returns legacy error.+- [x] set operation inside derived table preserves derived output lineage.+- [x] set operation inside CTE preserves CTE output lineage.+- [x] parenthesized set operation preserves legacy grouping behavior.++## Phase 5: Access Tables, System Tables, And Errors++### 5.1 Access Tables++- [x] top-level `FROM` table appears in `SourceColumns` as table-level access.+- [x] joined tables appear in `SourceColumns` as table-level access.+- [x] derived-table body tables appear in access tables.+- [x] CTE body tables appear in access tables.+- [x] scalar subquery tables appear in access tables.+- [x] correlated subquery inner tables appear in access tables.+- [x] function arguments containing subqueries appear in access tables.+- [x] function-level `ORDER BY` subqueries appear in access tables.+- [x] `VALUES` expressions containing subqueries appear in access tables.+- [x] `SET` assignment expressions containing subqueries appear in access tables.+- [x] `SET` assignment expressions containing subqueries participate in mixed user/system table detection.+- [x] `TABLE ... ORDER BY` subqueries participate in mixed user/system table detection.+- [x] `TABLE ... LIMIT` subqueries participate in mixed user/system table detection when parsed.+- [x] `CALL` arguments containing subqueries participate in mixed user/system table detection.+- [x] `LOCK TABLES` references participate in mixed user/system table detection.+- [x] DML/DDL roots such as `INSERT ... SELECT` and `CREATE TABLE ... AS SELECT` participate in mixed user/system table detection.+- [x] table-bearing DDL roots such as `DROP TABLE` and `RENAME TABLE` participate in mixed user/system table detection.+- [x] plain `EXPLAIN` payload tables participate in mixed user/system table detection.+- [x] `HANDLER` table appears in access tables where legacy did.++### 5.2 System Tables++- [x] all-system select returns `SelectInfoSchema`.+- [x] all-system select suppresses returned access tables.+- [x] mixed user/system table query returns `MixUserSystemTablesError`.+- [x] uppercase system schema follows case-sensitive legacy behavior.+- [x] uppercase system schema follows case-insensitive legacy behavior.+- [x] `information_schema` access behaves like legacy system resource detection.+- [x] `performance_schema` access behaves like legacy system resource detection.+- [x] `mysql` schema access behaves like legacy system resource detection.++### 5.3 Error Semantics++- [x] missing database maps to fail-open `NotFoundError`.+- [x] missing table maps to fail-open `NotFoundError`.+- [x] missing column maps to fail-open `NotFoundError`.+- [x] unsupported select-with-into returns the legacy hard error.+- [x] unsupported table-with-into returns the legacy hard error.+- [x] `EXPLAIN ANALYZE` does not require output lineage schema lookups for stale metadata.+- [ ] unsupported table source returns a hard error instead of silent empty lineage.+- [x] unsupported expression node cannot silently return empty source columns.+- [x] parser failure returns `parser/base.SyntaxError` with position and does not fabricate a query span.+- [x] nil or empty statement list returns the legacy empty select span.++## Phase 6: Metadata, Views, Engines, And Case Sensitivity++### 6.1 Metadata And Views++- [x] view column lineage is derived from the view definition.+- [x] view output column names are applied to view-derived results.+- [x] view recursion through nested views preserves source table lineage.+- [x] missing view dependency maps to legacy `NotFoundError` behavior.+- [x] duplicate database names with case differences follow `ignoreCaseSensitive`.+- [x] default database is used when table reference omits database.+- [x] explicit database overrides default database.++### 6.2 Case Sensitivity++- [x] case-sensitive column lookup preserves exact table/database matching rules.+- [x] case-insensitive column lookup resolves case variants.+- [x] table alias matching follows legacy case-sensitivity behavior.+- [x] CTE name matching follows legacy case-sensitivity behavior.+- [x] derived table alias matching follows legacy case-sensitivity behavior.+- [x] quoted identifier case is preserved in result names where legacy preserves it.++### 6.3 Engine-Specific Behavior++- [x] current StarRocks fixtures keep matching golden output.+- [x] StarRocks cluster-qualified database names are normalized before metadata lookup.+- [x] StarRocks system/user table mixing follows legacy behavior.+- [x] MySQL and StarRocks use the same lineage rules unless a fixture proves otherwise.