Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(pg): migrate completion ANTLR usage to omni parser [BYT-9010] (#19747)
rebelice7f685f5c8d31
diff --git a/backend/plugin/parser/pg/completion.go b/backend/plugin/parser/pg/completion.goindex bcc553bf94..0682c62980 100644--- a/backend/plugin/parser/pg/completion.go+++ b/backend/plugin/parser/pg/completion.go@@ -8,9 +8,10 @@ import ("unicode""unicode/utf8"- "github.com/antlr4-go/antlr/v4"+ omnipg "github.com/bytebase/omni/pg"+ "github.com/bytebase/omni/pg/ast"pgparser "github.com/bytebase/omni/pg/parser"- pg "github.com/bytebase/parser/postgresql"+ pgantlr "github.com/bytebase/parser/postgresql""github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -269,7 +270,7 @@ func (m CompletionMap) Insert(entry base.Candidate) {}func (m CompletionMap) insertFunctions() {- for _, name := range pg.GetBuiltinFunctions() {+ for _, name := range pgantlr.GetBuiltinFunctions() {m.Insert(base.Candidate{Type: base.CandidateTypeFunction,Text: name + "()",@@ -768,59 +769,81 @@ func (c *Completer) extractCTETables(pos int) []*base.VirtualTableReference {return nil}- input := antlr.NewInputStream(followingText)- lexer := pg.NewPostgreSQLLexer(input)- tokens := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- parser := pg.NewPostgreSQLParser(tokens)-- parser.BuildParseTrees = true- parser.RemoveErrorListeners()- lexer.RemoveErrorListeners()- tree := parser.With_clause()-- listener := &CTETableListener{context: c}- antlr.ParseTreeWalkerDefault.Walk(listener, tree)-- c.cteCache[pos] = listener.tables- return listener.tables-}--type CTETableListener struct {- *pg.BasePostgreSQLParserListener-- context *Completer- tables []*base.VirtualTableReference-}--func (l *CTETableListener) EnterCommon_table_expr(ctx *pg.Common_table_exprContext) {- table := &base.VirtualTableReference{}- if ctx.Name() != nil {- table.Table = normalizePostgreSQLName(ctx.Name())+ // Wrap the WITH fragment into a complete statement so omni can parse it.+ // The followingText may contain an incomplete main query after the CTE definitions,+ // so try progressively shorter token prefixes until parsing succeeds.+ const suffix = " SELECT 1"+ tokens := pgparser.Tokenize(followingText)++ var selStmt *ast.SelectStmt+ var wrappedSQL string+ for end := len(tokens); end > 0; end-- {+ if end < len(tokens) {+ lastTok := tokens[end-1]+ wrappedSQL = followingText[:lastTok.End] + suffix+ } else {+ wrappedSQL = followingText + suffix+ }+ stmts, err := omnipg.Parse(wrappedSQL)+ if err != nil || len(stmts) == 0 {+ continue+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if ok && sel.WithClause != nil && sel.WithClause.Ctes != nil {+ selStmt = sel+ break+ }+ }+ if selStmt == nil {+ c.cteCache[pos] = nil+ return nil}- if ctx.Opt_name_list() != nil {- for _, column := range ctx.Opt_name_list().Name_list().AllName() {- table.Columns = append(table.Columns, normalizePostgreSQLName(column))++ var tables []*base.VirtualTableReference+ for _, item := range selStmt.WithClause.Ctes.Items {+ cte, ok := item.(*ast.CommonTableExpr)+ if !ok {+ continue}- } else {- if span, err := GetQuerySpan(- l.context.ctx,- base.GetQuerySpanContext{- InstanceID: l.context.instanceID,- GetDatabaseMetadataFunc: l.context.getMetadata,- ListDatabaseNamesFunc: l.context.listDatabaseNames,- },- base.Statement{Text: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Preparablestmt())},- l.context.defaultDatabase,- "",- false,- ); err == nil && span.NotFoundError == nil {- for _, column := range span.Results {- table.Columns = append(table.Columns, column.Name)+ table := &base.VirtualTableReference{+ Table: cte.Ctename,+ }+ if cte.Aliascolnames != nil && len(cte.Aliascolnames.Items) > 0 {+ for _, nameItem := range cte.Aliascolnames.Items {+ if s, ok := nameItem.(*ast.String); ok {+ table.Columns = append(table.Columns, s.Str)+ }+ }+ } else if cte.Ctequery != nil {+ // Extract the CTE query text from the wrapped SQL using Loc.+ var queryText string+ if sel, ok := cte.Ctequery.(*ast.SelectStmt); ok && sel.Loc.Start >= 0 && sel.Loc.End > sel.Loc.Start && sel.Loc.End <= len(wrappedSQL) {+ queryText = wrappedSQL[sel.Loc.Start:sel.Loc.End]+ }+ if queryText != "" {+ if span, err := GetQuerySpan(+ c.ctx,+ base.GetQuerySpanContext{+ InstanceID: c.instanceID,+ GetDatabaseMetadataFunc: c.getMetadata,+ ListDatabaseNamesFunc: c.listDatabaseNames,+ },+ base.Statement{Text: queryText},+ c.defaultDatabase,+ "",+ false,+ ); err == nil && span.NotFoundError == nil {+ for _, column := range span.Results {+ table.Columns = append(table.Columns, column.Name)+ }+ }}}+ tables = append(tables, table)}- l.tables = append(l.tables, table)+ c.cteCache[pos] = tables+ return tables}// isInAliasAllowedContext checks if the caret is in a context where SELECT item@@ -878,33 +901,21 @@ func (c *Completer) extractAliasText(pos int) string {return ""}- input := antlr.NewInputStream(followingText)- lexer := pg.NewPostgreSQLLexer(input)- tokens := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- parser := pg.NewPostgreSQLParser(tokens)-- parser.BuildParseTrees = true- parser.RemoveErrorListeners()- lexer.RemoveErrorListeners()- tree := parser.Target_alias()-- listener := &TargetAliasListener{}- antlr.ParseTreeWalkerDefault.Walk(listener, tree)- return listener.result-}--type TargetAliasListener struct {- *pg.BasePostgreSQLParserListener-- result string-}--func (l *TargetAliasListener) EnterTarget_alias(ctx *pg.Target_aliasContext) {- if ctx.Bare_col_label() != nil {- l.result = normalizePostgreSQLBareColLabel(ctx.Bare_col_label())- } else if ctx.Collabel() != nil {- l.result = normalizePostgreSQLCollabel(ctx.Collabel())+ // The alias position may point to AS keyword or directly to the identifier.+ // Use the omni tokenizer to extract the alias name.+ tokens := pgparser.Tokenize(followingText)+ if len(tokens) == 0 {+ return ""}+ idx := 0+ // Skip the AS keyword if present.+ if tokens[0].Type == pgparser.AS && len(tokens) > 1 {+ idx = 1+ }+ if idx >= len(tokens) || !pgparser.IsIdentifierTokenType(tokens[idx].Type) {+ return ""+ }+ return normalizeIdentifier(tokens[idx].Str)}type ObjectFlags int@@ -1099,89 +1110,90 @@ func (c *Completer) collectLeadingTableReferences(caretIndex int) {}func (c *Completer) parseTableReferences(fromClause string) {- input := antlr.NewInputStream(fromClause)- lexer := pg.NewPostgreSQLLexer(input)- tokens := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- parser := pg.NewPostgreSQLParser(tokens)-- parser.BuildParseTrees = true- parser.RemoveErrorListeners()- lexer.RemoveErrorListeners()- tree := parser.From_clause()-- listener := &TableRefListener{- context: c,+ // Wrap the FROM fragment into a complete SELECT statement for omni parsing.+ // The fragment may be incomplete (user is still typing), so try progressively+ // shorter token prefixes until we find one that parses.+ const prefix = "SELECT * "+ tokens := pgparser.Tokenize(fromClause)++ var selStmt *ast.SelectStmt+ var wrappedSQL string+ for end := len(tokens); end > 0; end-- {+ if end < len(tokens) {+ lastTok := tokens[end-1]+ wrappedSQL = prefix + fromClause[:lastTok.End]+ } else {+ wrappedSQL = prefix + fromClause+ }+ stmts, err := omnipg.Parse(wrappedSQL)+ if err != nil || len(stmts) == 0 {+ continue+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if ok && sel.FromClause != nil {+ selStmt = sel+ break+ }+ }+ if selStmt == nil {+ return}- antlr.ParseTreeWalkerDefault.Walk(listener, tree)-}--type TableRefListener struct {- *pg.BasePostgreSQLParserListener- context *Completer- level int+ for _, item := range selStmt.FromClause.Items {+ c.extractFromItem(item, wrappedSQL)+ }}-func (l *TableRefListener) EnterTable_ref(ctx *pg.Table_refContext) {- if _, ok := ctx.GetParent().(*pg.Table_refContext); ok {- // if the table reference is nested, we should not process it.- l.level++- }-- if l.level == 0 {- switch {- case ctx.Relation_expr() != nil:- var reference base.TableReference- physicalReference := &base.PhysicalTableReference{}- // We should use the physical reference as the default reference.- reference = physicalReference- list := NormalizePostgreSQLQualifiedName(ctx.Relation_expr().Qualified_name())- switch len(list) {- case 1:- physicalReference.Table = list[0]- case 2:- physicalReference.Schema = list[0]- physicalReference.Table = list[1]- case 3:- physicalReference.Database = list[0]- physicalReference.Schema = list[1]- physicalReference.Table = list[2]- default:- return- }-- if ctx.Opt_alias_clause() != nil {- tableAlias, columnAlias := normalizeTableAlias(ctx.Opt_alias_clause())- if len(columnAlias) > 0 {- virtualReference := &base.VirtualTableReference{- Table: tableAlias,- Columns: columnAlias,- }- // If the table alias has the column alias, we should use the virtual reference.- reference = virtualReference- } else {- physicalReference.Alias = tableAlias+// extractFromItem extracts table references from a single FROM clause item.+// It only processes top-level references (not nested JOINs' subqueries).+func (c *Completer) extractFromItem(item ast.Node, wrappedSQL string) {+ switch v := item.(type) {+ case *ast.RangeVar:+ var reference base.TableReference+ physicalReference := &base.PhysicalTableReference{}+ reference = physicalReference++ // omni already normalizes identifiers: lowercase for unquoted, preserve case for quoted.+ physicalReference.Database = v.Catalogname+ physicalReference.Schema = v.Schemaname+ physicalReference.Table = v.Relname++ if v.Alias != nil {+ tableAlias := v.Alias.Aliasname+ columnAlias := extractAliasColnames(v.Alias)+ if len(columnAlias) > 0 {+ reference = &base.VirtualTableReference{+ Table: tableAlias,+ Columns: columnAlias,}+ } else {+ physicalReference.Alias = tableAlias}+ }- l.context.referencesStack[0] = append(l.context.referencesStack[0], reference)- case ctx.Select_with_parens() != nil:- if ctx.Opt_alias_clause() != nil {- virtualReference := &base.VirtualTableReference{}- tableAlias, columnAlias := normalizeTableAlias(ctx.Opt_alias_clause())- virtualReference.Table = tableAlias- if len(columnAlias) > 0 {- virtualReference.Columns = columnAlias- } else {+ c.referencesStack[0] = append(c.referencesStack[0], reference)++ case *ast.RangeSubselect:+ if v.Alias != nil {+ virtualReference := &base.VirtualTableReference{}+ tableAlias := v.Alias.Aliasname+ virtualReference.Table = tableAlias+ columnAlias := extractAliasColnames(v.Alias)+ if len(columnAlias) > 0 {+ virtualReference.Columns = columnAlias+ } else if v.Subquery != nil {+ // Extract subquery text from wrappedSQL using Loc.+ if sel, ok := v.Subquery.(*ast.SelectStmt); ok && sel.Loc.Start >= 0 && sel.Loc.End > sel.Loc.Start && sel.Loc.End <= len(wrappedSQL) {+ subqueryText := wrappedSQL[sel.Loc.Start:sel.Loc.End]if span, err := GetQuerySpan(- l.context.ctx,+ c.ctx,base.GetQuerySpanContext{- InstanceID: l.context.instanceID,- GetDatabaseMetadataFunc: l.context.getMetadata,- ListDatabaseNamesFunc: l.context.listDatabaseNames,+ InstanceID: c.instanceID,+ GetDatabaseMetadataFunc: c.getMetadata,+ ListDatabaseNamesFunc: c.listDatabaseNames,},- base.Statement{Text: fmt.Sprintf("SELECT * FROM %s AS %s;", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Select_with_parens()), tableAlias)},- l.context.defaultDatabase,+ base.Statement{Text: fmt.Sprintf("SELECT * FROM (%s) AS %s;", subqueryText, tableAlias)},+ c.defaultDatabase,"",false,); err == nil && span.NotFoundError == nil {@@ -1190,57 +1202,53 @@ func (l *TableRefListener) EnterTable_ref(ctx *pg.Table_refContext) {}}}+ }+ c.referencesStack[0] = append(c.referencesStack[0], virtualReference)+ }- l.context.referencesStack[0] = append(l.context.referencesStack[0], virtualReference)+ case *ast.JoinExpr:+ if v.Alias != nil {+ // Parenthesized JOIN with alias: FROM (t1 JOIN t2 USING (c1)) AS j(ca, cb)+ virtualReference := &base.VirtualTableReference{+ Table: v.Alias.Aliasname,+ Columns: extractAliasColnames(v.Alias),}- case ctx.OPEN_PAREN() != nil:- if ctx.Opt_alias_clause() != nil {- virtualReference := &base.VirtualTableReference{}- tableAlias, columnAlias := normalizeTableAlias(ctx.Opt_alias_clause())- virtualReference.Table = tableAlias- if len(columnAlias) > 0 {- virtualReference.Columns = columnAlias- }+ c.referencesStack[0] = append(c.referencesStack[0], virtualReference)+ } else {+ c.extractFromItem(v.Larg, wrappedSQL)+ c.extractFromItem(v.Rarg, wrappedSQL)+ }- l.context.referencesStack[0] = append(l.context.referencesStack[0], virtualReference)+ case *ast.RangeFunction:+ if v.Alias != nil {+ columnAlias := extractAliasColnames(v.Alias)+ if len(columnAlias) > 0 {+ c.referencesStack[0] = append(c.referencesStack[0], &base.VirtualTableReference{+ Table: v.Alias.Aliasname,+ Columns: columnAlias,+ })}- default:- // Other cases+ // Functions without explicit column aliases (e.g. generate_series(1,10) g)+ // are not added as references since their return columns are unknown.}- }-}-func (l *TableRefListener) ExitTable_ref(ctx *pg.Table_refContext) {- if _, ok := ctx.GetParent().(*pg.Table_refContext); ok {- l.level--+ default:}}-func (l *TableRefListener) EnterSelect_with_parens(_ *pg.Select_with_parensContext) {- l.level++-}--func (l *TableRefListener) ExitSelect_with_parens(_ *pg.Select_with_parensContext) {- l.level---}--func normalizeTableAlias(ctx pg.IOpt_alias_clauseContext) (string, []string) {- if ctx == nil || ctx.Table_alias_clause() == nil {- return "", nil- }-- tableAlias := ""- aliasClause := ctx.Table_alias_clause()- if aliasClause.Table_alias() != nil {- tableAlias = normalizePostgreSQLTableAlias(aliasClause.Table_alias())+// extractAliasColnames extracts column alias names from an ast.Alias.+func extractAliasColnames(alias *ast.Alias) []string {+ if alias == nil || alias.Colnames == nil {+ return nil}-- var columnAliases []string- if aliasClause.Name_list() != nil {- columnAliases = append(columnAliases, normalizePostgreSQLNameList(aliasClause.Name_list())...)+ var result []string+ for _, item := range alias.Colnames.Items {+ if s, ok := item.(*ast.String); ok {+ // omni already normalizes: lowercase for unquoted, preserve case for quoted.+ result = append(result, s.Str)+ }}-- return tableAlias, columnAliases+ return result}// caretLine is 1-based and caretOffset is 0-based.diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.goindex 853ed33e61..1c9993ae60 100644--- a/backend/plugin/parser/pg/pg.go+++ b/backend/plugin/parser/pg/pg.go@@ -170,33 +170,6 @@ func parseSinglePostgreSQL(sql string, baseLine int) (*base.ANTLRAST, error) {return result, nil}-func normalizePostgreSQLTableAlias(ctx parser.ITable_aliasContext) string {- if ctx == nil {- return ""- }-- switch {- case ctx.Identifier() != nil:- return normalizePostgreSQLIdentifier(ctx.Identifier())- default:- // For non-quote identifier, we just return the lower string for PostgreSQL.- return strings.ToLower(ctx.GetText())- }-}--func normalizePostgreSQLNameList(ctx parser.IName_listContext) []string {- if ctx == nil {- return nil- }-- var result []string- for _, item := range ctx.AllName() {- result = append(result, normalizePostgreSQLName(item))- }-- return result-}-func normalizePostgreSQLName(ctx parser.INameContext) string {if ctx == nil {return ""@@ -284,16 +257,6 @@ func normalizePostgreSQLCollabel(ctx parser.ICollabelContext) string {return strings.ToLower(ctx.GetText())}-func normalizePostgreSQLBareColLabel(ctx parser.IBare_col_labelContext) string {- if ctx == nil {- return ""- }- if ctx.Identifier() != nil {- return normalizePostgreSQLIdentifier(ctx.Identifier())- }- return strings.ToLower(ctx.GetText())-}-// NormalizePostgreSQLColid normalizes the given colid.func NormalizePostgreSQLColid(ctx parser.IColidContext) string {if ctx == nil {diff --git a/backend/plugin/parser/pg/test-data/test_completion.yaml b/backend/plugin/parser/pg/test-data/test_completion.yamlindex d0701b8e22..05334089fb 100644--- a/backend/plugin/parser/pg/test-data/test_completion.yaml+++ b/backend/plugin/parser/pg/test-data/test_completion.yaml@@ -3052,6 +3052,16 @@definition: public.t1 | int, NOT NULLcomment: ""priority: 0+ - text: c1+ type: COLUMN+ definition: public.t2 | int, NOT NULL+ comment: ""+ priority: 0+ - text: c2+ type: COLUMN+ definition: public.t2 | int, NOT NULL+ comment: ""+ priority: 0- text: ft1type: FOREIGNTABLEdefinition: ""