Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(pg): migrate backup and restore from ANTLR to omni parser (#19692)
rebeliced11a47a995e8
diff --git a/backend/plugin/parser/pg/backup.go b/backend/plugin/parser/pg/backup.goindex f65d460e3e..aa263d16f5 100644--- a/backend/plugin/parser/pg/backup.go+++ b/backend/plugin/parser/pg/backup.go@@ -6,10 +6,9 @@ import ("slices""strings"- "github.com/antlr4-go/antlr/v4""github.com/pkg/errors"- parser "github.com/bytebase/parser/postgresql"+ "github.com/bytebase/omni/pg/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -53,9 +52,11 @@ func (t *TableReference) String() string {type statementInfo struct {offset intstatement string- tree antlr.ParserRuleContext+ node ast.Nodetable *TableReference- baseLine int+ startPos *storepb.Position+ endPos *storepb.Position+ fullSQL string}func init() {@@ -136,7 +137,7 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,}// In PostgreSQL, WITH clause must appear once before the first SELECT.- cteClause := extractCTE(statementInfoList[0].tree)+ cteClause := extractCTE(statementInfoList[0].node, statementInfoList[0].fullSQL)if cteClause != "" {if _, err := fmt.Fprintf(&buf, "%s\n", cteClause); err != nil {return nil, errors.Wrap(err, "failed to write to buffer")@@ -159,7 +160,7 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,}}- if err := writeSuffixSelectClause(&buf, item.tree); err != nil {+ if err := writeSuffixSelectClause(&buf, item.node, item.fullSQL); err != nil {return nil, errors.Wrap(err, "failed to write string with new line")}}@@ -173,98 +174,106 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,SourceSchema: table.Schema,SourceTableName: table.Table,TargetTableName: targetTable,- StartPosition: &storepb.Position{- Line: int32(statementInfoList[0].tree.GetStart().GetLine() + statementInfoList[0].baseLine),- Column: int32(statementInfoList[0].tree.GetStart().GetColumn()),- },- EndPosition: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetLine()+statementInfoList[len(statementInfoList)-1].baseLine),- int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetColumn()),- statementInfoList[len(statementInfoList)-1].tree.GetStop().GetText(),- ),+ StartPosition: statementInfoList[0].startPos,+ EndPosition: statementInfoList[len(statementInfoList)-1].endPos,}, nil}-func extractCTE(ctx antlr.ParserRuleContext) string {- switch node := ctx.(type) {- case *parser.UpdatestmtContext:- if optWith := node.Opt_with_clause(); optWith != nil {- if withClause := optWith.With_clause(); withClause != nil {- return node.GetParser().GetTokenStream().GetTextFromRuleContext(withClause)- }- }- case *parser.DeletestmtContext:- if optWith := node.Opt_with_clause(); optWith != nil {- if withClause := optWith.With_clause(); withClause != nil {- return node.GetParser().GetTokenStream().GetTextFromRuleContext(withClause)- }- }+func extractCTE(node ast.Node, fullSQL string) string {+ var wc *ast.WithClause+ switch n := node.(type) {+ case *ast.UpdateStmt:+ wc = n.WithClause+ case *ast.DeleteStmt:+ wc = n.WithClausedefault:}- return ""-}--func writeSuffixSelectClause(buf *strings.Builder, tree antlr.Tree) error {- extractor := &suffixSelectClauseExtractor{- buf: buf,+ if wc == nil {+ return ""}- antlr.ParseTreeWalkerDefault.Walk(extractor, tree)- return extractor.err+ return extractNodeText(wc.Loc, fullSQL)}-type suffixSelectClauseExtractor struct {- *parser.BasePostgreSQLParserListener-- buf *strings.Builder- err error+func writeSuffixSelectClause(buf *strings.Builder, node ast.Node, fullSQL string) error {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ return writeUpdateSuffix(buf, n, fullSQL)+ case *ast.DeleteStmt:+ return writeDeleteSuffix(buf, n, fullSQL)+ }+ return nil}-func (e *suffixSelectClauseExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {- if e.err != nil || !isTopLevel(ctx.GetParent()) {- return+func writeUpdateSuffix(buf *strings.Builder, n *ast.UpdateStmt, sql string) error {+ if n.Relation == nil {+ return nil}- if _, err := fmt.Fprintf(e.buf, "FROM %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Relation_expr_opt_alias())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ relText := extractNodeText(n.Relation.Loc, sql)+ if _, err := fmt.Fprintf(buf, "FROM %s", relText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")}- if ctx.From_clause() != nil {- if _, err := fmt.Fprintf(e.buf, ", %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.From_clause().From_list())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ if n.FromClause != nil && n.FromClause.Len() > 0 {+ fromText := extractNodeText(ast.ListSpan(n.FromClause), sql)+ if fromText != "" {+ if _, err := fmt.Fprintf(buf, ", %s", fromText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")+ }}}- if ctx.Where_or_current_clause() != nil {- if _, err := fmt.Fprintf(e.buf, " %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_or_current_clause())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ if n.WhereClause != nil {+ whereText := extractNodeText(ast.NodeLoc(n.WhereClause), sql)+ if whereText != "" {+ if _, err := fmt.Fprintf(buf, " WHERE %s", whereText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")+ }}}+ return nil}-func (e *suffixSelectClauseExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {- if e.err != nil || !isTopLevel(ctx.GetParent()) {- return+func writeDeleteSuffix(buf *strings.Builder, n *ast.DeleteStmt, sql string) error {+ if n.Relation == nil {+ return nil}- if _, err := fmt.Fprintf(e.buf, "FROM %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Relation_expr_opt_alias())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ relText := extractNodeText(n.Relation.Loc, sql)+ if _, err := fmt.Fprintf(buf, "FROM %s", relText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")}- if ctx.Using_clause() != nil {- if _, err := fmt.Fprintf(e.buf, ", %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Using_clause().From_list())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ if n.UsingClause != nil && n.UsingClause.Len() > 0 {+ usingText := extractNodeText(ast.ListSpan(n.UsingClause), sql)+ if usingText != "" {+ if _, err := fmt.Fprintf(buf, ", %s", usingText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")+ }}}- if ctx.Where_or_current_clause() != nil {- if _, err := fmt.Fprintf(e.buf, " %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_or_current_clause())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")+ if n.WhereClause != nil {+ whereText := extractNodeText(ast.NodeLoc(n.WhereClause), sql)+ if whereText != "" {+ if _, err := fmt.Fprintf(buf, " WHERE %s", whereText); err != nil {+ return errors.Wrap(err, "failed to write to buffer")+ }}}+ return nil+}++// extractNodeText extracts trimmed text from SQL using a Loc range.+func extractNodeText(loc ast.Loc, sql string) string {+ if loc.Start < 0 || loc.End < 0 || loc.Start >= loc.End || loc.End > len(sql) {+ return ""+ }+ return strings.TrimSpace(sql[loc.Start:loc.End])}func prepareTransformation(ctx context.Context, tCtx base.TransformContext, statement string) ([]statementInfo, error) {- parseResults, err := ParsePostgreSQL(statement)+ stmts, err := ParsePg(statement)if err != nil {return nil, errors.Wrap(err, "failed to parse statement")}@@ -278,178 +287,111 @@ func prepareTransformation(ctx context.Context, tCtx base.TransformContext, statreturn nil, errors.Wrap(err, "failed to get database metadata")}- extractor := &dmlExtractor{- metadata: metadata,- searchPath: metadata.GetSearchPath(),- }+ searchPath := metadata.GetSearchPath()+ var dmls []statementInfo- // Walk all parse results to extract DML statements- for _, parseResult := range parseResults {- extractor.currentBaseLine = base.GetLineOffset(parseResult.StartPosition)- antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)- if extractor.err != nil {- return nil, extractor.err+ for i, stmt := range stmts {+ if stmt.Empty() {+ continue}- }-- return extractor.dmls, nil-}-type dmlExtractor struct {- *parser.BasePostgreSQLParserListener-- metadata *model.DatabaseMetadata- searchPath []string- dmls []statementInfo- offset int- err error- currentBaseLine int-}--func isTopLevel(ctx antlr.Tree) bool {- if ctx == nil {- return true- }-- switch ctx := ctx.(type) {- case *parser.RootContext, *parser.StmtblockContext:- return true- case *parser.StmtmultiContext, *parser.StmtContext:- return isTopLevel(ctx.GetParent())- default:- return false- }-}--func (e *dmlExtractor) EnterVariablesetstmt(ctx *parser.VariablesetstmtContext) {- setRest := ctx.Set_rest()- if setRest == nil {- return- }- setRestMore := setRest.Set_rest_more()- if setRestMore == nil {- return- }- genericSet := setRestMore.Generic_set()- if genericSet == nil {- return- }- varName := genericSet.Var_name()- if varName == nil {- return- }- if len(varName.AllColid()) != 1 {- return- }- name := NormalizePostgreSQLColid(varName.Colid(0))- if !strings.EqualFold(name, "search_path") {- return- }- var searchPath []string- for _, value := range genericSet.Var_list().AllVar_value() {- valueText := value.GetText()- if strings.HasPrefix(valueText, "\"") && strings.HasSuffix(valueText, "\"") {- // Remove the quotes from the schema name.- valueText = strings.Trim(valueText, "\"")- } else if strings.HasPrefix(valueText, "'") && strings.HasSuffix(valueText, "'") {- // Remove the quotes from the schema name.- valueText = strings.Trim(valueText, "'")- } else {- // For non-quoted schema names, we just return the lower string for PostgreSQL.- valueText = strings.ToLower(valueText)- }- searchPath = append(searchPath, strings.TrimSpace(valueText))- }- e.searchPath = searchPath-}--func (e *dmlExtractor) ExitStmt(ctx *parser.StmtContext) {- if isTopLevel(ctx) {- e.offset++- }-}--func (e *dmlExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {- if isTopLevel(ctx.GetParent()) {- table, err := e.extractTableReference(ctx.Relation_expr_opt_alias())- if err != nil {- e.err = errors.Wrapf(err, "failed to extract table reference from update statement at offset %d", e.offset)- return- }- if table == nil {- return+ switch n := stmt.AST.(type) {+ case *ast.VariableSetStmt:+ if strings.EqualFold(n.Name, "search_path") && n.Args != nil {+ var newSearchPath []string+ for _, arg := range n.Args.Items {+ if ac, ok := arg.(*ast.A_Const); ok {+ if s, ok := ac.Val.(*ast.String); ok {+ newSearchPath = append(newSearchPath, s.Str)+ }+ }+ }+ if len(newSearchPath) > 0 {+ searchPath = newSearchPath+ }+ }+ case *ast.UpdateStmt:+ table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to extract table reference from update statement at offset %d", i)+ }+ if table == nil {+ continue+ }+ table.StatementType = StatementTypeUpdate+ nodeLoc := n.Loc+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)+ startPos.Column-- // 0-based column to match existing convention+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)+ dmls = append(dmls, statementInfo{+ offset: i,+ statement: stmt.Text,+ node: n,+ table: table,+ startPos: startPos,+ endPos: endPos,+ fullSQL: statement,+ })+ case *ast.DeleteStmt:+ table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to extract table reference from delete statement at offset %d", i)+ }+ if table == nil {+ continue+ }+ table.StatementType = StatementTypeDelete+ nodeLoc := n.Loc+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)+ startPos.Column--+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)+ dmls = append(dmls, statementInfo{+ offset: i,+ statement: stmt.Text,+ node: n,+ table: table,+ startPos: startPos,+ endPos: endPos,+ fullSQL: statement,+ })+ default:}- table.StatementType = StatementTypeUpdate- e.dmls = append(e.dmls, statementInfo{- offset: e.offset,- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- tree: ctx,- table: table,- baseLine: e.currentBaseLine,- })}-}-func (e *dmlExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {- if isTopLevel(ctx.GetParent()) {- table, err := e.extractTableReference(ctx.Relation_expr_opt_alias())- if err != nil {- e.err = errors.Wrapf(err, "failed to extract table reference from delete statement at offset %d", e.offset)- return- }- if table == nil {- return- }- table.StatementType = StatementTypeDelete- e.dmls = append(e.dmls, statementInfo{- offset: e.offset,- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- tree: ctx,- table: table,- baseLine: e.currentBaseLine,- })- }+ return dmls, nil}-func (e *dmlExtractor) extractTableReference(ctx parser.IRelation_expr_opt_aliasContext) (*TableReference, error) {- if ctx == nil {+func extractTableReferenceFromRangeVar(rv *ast.RangeVar, metadata *model.DatabaseMetadata, searchPath []string) (*TableReference, error) {+ if rv == nil {return nil, nil}table := TableReference{}- relationExpr := ctx.Relation_expr()- if relationExpr == nil {- return nil, nil- }-- list := NormalizePostgreSQLQualifiedName(relationExpr.Qualified_name())- switch len(list) {- case 3:- table.Database = list[0]- table.Schema = list[1]- table.Table = list[2]- case 2:- table.Schema = list[0]- table.Table = list[1]- case 1:+ switch {+ case rv.Catalogname != "":+ table.Database = rv.Catalogname+ table.Schema = rv.Schemaname+ table.Table = rv.Relname+ case rv.Schemaname != "":+ table.Schema = rv.Schemaname+ table.Table = rv.Relname+ default:// TODO: remove it in the future.// Handle the case where the search path is not synchronized with the metadata.- if len(e.searchPath) == 0 {- e.searchPath = []string{"public"}+ if len(searchPath) == 0 {+ searchPath = []string{"public"}}- schemaName, _ := e.metadata.SearchObject(e.searchPath, list[0])+ schemaName, _ := metadata.SearchObject(searchPath, rv.Relname)if schemaName == "" {- return nil, errors.Errorf("Table %q not found in metadata with search path %v", list[0], e.searchPath)+ return nil, errors.Errorf("Table %q not found in metadata with search path %v", rv.Relname, searchPath)}table.Schema = schemaName- table.Table = list[0]- default:- return nil, errors.Errorf("Invalid table name: %v", list)+ table.Table = rv.Relname}- if ctx.Colid() != nil {- table.Alias = NormalizePostgreSQLColid(ctx.Colid())+ if rv.Alias != nil && rv.Alias.Aliasname != "" {+ table.Alias = rv.Alias.Aliasname}return &table, nil}diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.goindex 24e1d56ba8..84af08edaf 100644--- a/backend/plugin/parser/pg/pg.go+++ b/backend/plugin/parser/pg/pg.go@@ -281,27 +281,6 @@ func NormalizePostgreSQLQualifiedName(ctx parser.IQualified_nameContext) []strinreturn res}-func normalizePostgreSQLSetTarget(ctx parser.ISet_targetContext) []string {- if ctx == nil {- return []string{}- }-- var res []string- res = append(res, NormalizePostgreSQLColid(ctx.Colid()))- res = append(res, normalizePostgreSQLOptIndirection(ctx.Opt_indirection())...)- return res-}--func normalizePostgreSQLOptIndirection(ctx parser.IOpt_indirectionContext) []string {- var res []string- for _, child := range ctx.AllIndirection_el() {- if part := normalizePostgreSQLIndirectionEl(child); part != "" {- res = append(res, part)- }- }- return res-}-func normalizePostgreSQLIndirection(ctx parser.IIndirectionContext) []string {if ctx == nil {return []string{}diff --git a/backend/plugin/parser/pg/restore.go b/backend/plugin/parser/pg/restore.goindex 47fe06f3fa..02c1dd4039 100644--- a/backend/plugin/parser/pg/restore.go+++ b/backend/plugin/parser/pg/restore.go@@ -5,10 +5,9 @@ import ("fmt""strings"- "github.com/antlr4-go/antlr/v4""github.com/pkg/errors"- parser "github.com/bytebase/parser/postgresql"+ "github.com/bytebase/omni/pg/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -34,28 +33,12 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statementreturn "", errors.Errorf("no original SQL")}- parseResults, err := ParsePostgreSQL(statement)+ // Find the matching DML statement node at the backup position.+ matchingNode, err := findStatementAtPosition(statement, backupItem)if err != nil {return "", err}-- // Find the parse result that contains the statement at the backup position- var targetResult *base.ANTLRAST- for _, parseResult := range parseResults {- // Walk the tree to find if this parse result contains the target statement- finder := &statementAtPositionFinder{- startPos: backupItem.StartPosition,- endPos: backupItem.EndPosition,- baseLine: base.GetLineOffset(parseResult.StartPosition),- }- antlr.ParseTreeWalkerDefault.Walk(finder, parseResult.Tree)- if finder.found {- targetResult = parseResult- break- }- }-- if targetResult == nil {+ if matchingNode == nil {return "", errors.Errorf("could not find statement at position (line %d:%d - %d:%d)",backupItem.StartPosition.Line, backupItem.StartPosition.Column,backupItem.EndPosition.Line, backupItem.EndPosition.Column)@@ -71,10 +54,35 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statementreturn "", errors.Wrap(err, "failed to get prepend statements")}- return doGenerate(ctx, rCtx, sqlForComment, targetResult, backupItem, prependStatements)+ return doGenerate(ctx, rCtx, sqlForComment, matchingNode, backupItem, prependStatements)+}++func findStatementAtPosition(statement string, backupItem *storepb.PriorBackupDetail_Item) (ast.Node, error) {+ stmts, err := ParsePg(statement)+ if err != nil {+ return nil, errors.Wrap(err, "failed to parse statement")+ }++ for _, stmt := range stmts {+ if stmt.Empty() {+ continue+ }+ switch stmt.AST.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ nodeLoc := ast.NodeLoc(stmt.AST)+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)+ startPos.Column-- // 0-based to match backup convention+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)+ if inRange(startPos, endPos, backupItem.StartPosition, backupItem.EndPosition) {+ return stmt.AST, nil+ }+ default:+ }+ }+ return nil, nil}-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, tree *base.ANTLRAST, backupItem *storepb.PriorBackupDetail_Item, prependStatements string) (string, error) {+func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node ast.Node, backupItem *storepb.PriorBackupDetail_Item, prependStatements string) (string, error) {_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)if err != nil {return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)@@ -107,19 +115,42 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment strreturn "", errors.Errorf("table metadata not found for %s.%s", schema, backupItem.SourceTable.Table)}- g := &generator{- ctx: ctx,- rCtx: rCtx,- backupSchema: backupItem.TargetTable.Schema,- backupTable: backupItem.TargetTable.Table,- originalSchema: schema,- originalTable: backupItem.SourceTable.Table,- table: tableMetadata,- isFirst: true,- }- antlr.ParseTreeWalkerDefault.Walk(g, tree.Tree)- if g.err != nil {- return "", g.err+ backupSchema := backupItem.TargetTable.Schema+ backupTable := backupItem.TargetTable.Table+ originalSchema := schema+ originalTable := backupItem.SourceTable.Table++ var result string+ switch n := node.(type) {+ case *ast.DeleteStmt:+ result = fmt.Sprintf(`INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s";`, originalSchema, originalTable, backupSchema, backupTable)+ case *ast.UpdateStmt:+ fields := extractSetFieldNames(n)+ uk, err := findDisjointUniqueKey(tableMetadata, fields)+ if err != nil {+ return "", err+ }++ var buf strings.Builder+ if _, err := fmt.Fprintf(&buf, `INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s" ON CONFLICT ON CONSTRAINT "%s" DO UPDATE SET `, originalSchema, originalTable, backupSchema, backupTable, uk); err != nil {+ return "", errors.Wrapf(err, "failed to generate update statement")+ }+ for i, field := range fields {+ if i > 0 {+ if _, err := fmt.Fprint(&buf, ", "); err != nil {+ return "", errors.Wrapf(err, "failed to generate update statement")+ }+ }+ if _, err := fmt.Fprintf(&buf, `"%s" = EXCLUDED."%s"`, field, field); err != nil {+ return "", errors.Wrapf(err, "failed to generate update statement")+ }+ }+ if _, err := fmt.Fprint(&buf, `;`); err != nil {+ return "", errors.Wrapf(err, "failed to generate update statement")+ }+ result = buf.String()+ default:+ return "", errors.Errorf("unexpected statement type: %T", node)}if len(prependStatements) > 0 {@@ -127,32 +158,42 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment strif !strings.HasSuffix(prependStatements, ";") {prependStatements += ";"}- return fmt.Sprintf("%s\n/*\nOriginal SQL:\n%s\n*/\n%s", prependStatements, sqlForComment, g.result), nil+ return fmt.Sprintf("%s\n/*\nOriginal SQL:\n%s\n*/\n%s", prependStatements, sqlForComment, result), nil}- return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result), nil-}--type generator struct {- *parser.BasePostgreSQLParserListener-- backupSchema string- backupTable string- originalSchema string- originalTable string- table *model.TableMetadata-- isFirst bool- ctx context.Context- rCtx base.RestoreContext- result string- err error+ return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, result), nil}-func (g *generator) EnterDeletestmt(ctx *parser.DeletestmtContext) {- if isTopLevel(ctx.GetParent()) && g.isFirst {- g.isFirst = false- g.result = fmt.Sprintf(`INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s";`, g.originalSchema, g.originalTable, g.backupSchema, g.backupTable)+// extractSetFieldNames extracts the target column names from an UPDATE SET clause.+func extractSetFieldNames(stmt *ast.UpdateStmt) []string {+ if stmt.TargetList == nil {+ return nil+ }+ var fields []string+ for _, item := range stmt.TargetList.Items {+ rt, ok := item.(*ast.ResTarget)+ if !ok || rt.Name == "" {+ continue+ }+ // Take the last name component (the actual column name).+ // For "SET test.c1 = 1", Name="test", Indirection=["c1"] → want "c1".+ // For "SET c1 = 1", Name="c1", Indirection=nil → want "c1".+ // For "SET schema.col[1] = val", Indirection=["col", integer] → want "col".+ if rt.Indirection != nil && rt.Indirection.Len() > 0 {+ found := false+ for j := rt.Indirection.Len() - 1; j >= 0; j-- {+ if s, ok := rt.Indirection.Items[j].(*ast.String); ok {+ fields = append(fields, s.Str)+ found = true+ break+ }+ }+ if found {+ continue+ }+ }+ fields = append(fields, rt.Name)}+ return fields}func disjoint(a []string, b map[string]bool) bool {@@ -164,18 +205,18 @@ func disjoint(a []string, b map[string]bool) bool {return true}-func (g *generator) findDisjointUniqueKey(fields []string) (string, error) {+func findDisjointUniqueKey(table *model.TableMetadata, fields []string) (string, error) {columnMap := make(map[string]bool)for _, field := range fields {columnMap[field] = true}- pk := g.table.GetPrimaryKey()+ pk := table.GetPrimaryKey()if pk != nil {if disjoint(pk.GetProto().Expressions, columnMap) {return pk.GetProto().Name, nil}}- for _, index := range g.table.GetProto().Indexes {+ for _, index := range table.GetProto().Indexes {if index.Primary {continue}@@ -187,61 +228,7 @@ func (g *generator) findDisjointUniqueKey(fields []string) (string, error) {}}- return "", errors.Errorf("no disjoint unique key found for %s.%s", g.originalSchema, g.originalTable)-}--func (g *generator) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {- if isTopLevel(ctx.GetParent()) && g.isFirst {- g.isFirst = false-- l := &setFieldListener{}- antlr.ParseTreeWalkerDefault.Walk(l, ctx)-- uk, err := g.findDisjointUniqueKey(l.result)- if err != nil {- g.err = err- return- }-- var buf strings.Builder- if _, err := fmt.Fprintf(&buf, `INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s" ON CONFLICT ON CONSTRAINT "%s" DO UPDATE SET `, g.originalSchema, g.originalTable, g.backupSchema, g.backupTable, uk); err != nil {- g.err = errors.Wrapf(err, "failed to generate update statement")- return- }- for i, field := range l.result {- if i > 0 {- if _, err := fmt.Fprint(&buf, ", "); err != nil {- g.err = errors.Wrapf(err, "failed to generate update statement")- return- }- }- // The field is written by user and no need to escape.- if _, err := fmt.Fprintf(&buf, `"%s" = EXCLUDED."%s"`, field, field); err != nil {- g.err = errors.Wrapf(err, "failed to generate update statement")- return- }- }- if _, err := fmt.Fprint(&buf, `;`); err != nil {- g.err = errors.Wrapf(err, "failed to generate update statement")- return- }- g.result = buf.String()- }-}--type setFieldListener struct {- *parser.BasePostgreSQLParserListener-- result []string-}--func (l *setFieldListener) EnterSet_clause(ctx *parser.Set_clauseContext) {- if ctx.Set_target() != nil {- names := normalizePostgreSQLSetTarget(ctx.Set_target())- if len(names) > 0 {- l.result = append(l.result, names[len(names)-1])- }- }+ return "", errors.Errorf("no disjoint unique key found for table")}func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {@@ -249,60 +236,32 @@ func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Itreturn "", errors.Errorf("backup item is nil")}- parseResults, err := ParsePostgreSQL(statement)+ stmts, err := ParsePg(statement)if err != nil {return "", errors.Wrapf(err, "failed to parse statement")}- l := &originalSQLExtractor{- startPos: backupItem.StartPosition,- endPos: backupItem.EndPosition,- }-- // Walk all parse results to find statements within the specified position range- for _, parseResult := range parseResults {- l.baseLine = base.GetLineOffset(parseResult.StartPosition)- antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)- }-- return strings.Join(l.originalSQL, ";\n"), nil-}--type originalSQLExtractor struct {- *parser.BasePostgreSQLParserListener-- originalSQL []string- startPos *storepb.Position- endPos *storepb.Position- baseLine int-}--func (l *originalSQLExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {- if isTopLevel(ctx.GetParent()) {- if inRange(&storepb.Position{- Line: int32(ctx.GetStart().GetLine()) + int32(l.baseLine),- Column: int32(ctx.GetStart().GetColumn()),- }, &storepb.Position{- Line: int32(ctx.GetStop().GetLine()) + int32(l.baseLine),- Column: int32(ctx.GetStop().GetColumn()),- }, l.startPos, l.endPos) {- l.originalSQL = append(l.originalSQL, ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx))+ var sqls []string+ for _, stmt := range stmts {+ if stmt.Empty() {+ continue}- }-}--func (l *originalSQLExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {- if isTopLevel(ctx.GetParent()) {- if inRange(&storepb.Position{- Line: int32(ctx.GetStart().GetLine()) + int32(l.baseLine),- Column: int32(ctx.GetStart().GetColumn()),- }, &storepb.Position{- Line: int32(ctx.GetStop().GetLine()) + int32(l.baseLine),- Column: int32(ctx.GetStop().GetColumn()),- }, l.startPos, l.endPos) {- l.originalSQL = append(l.originalSQL, ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx))+ switch stmt.AST.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ nodeLoc := ast.NodeLoc(stmt.AST)+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)+ startPos.Column-- // 0-based+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)+ if inRange(startPos, endPos, backupItem.StartPosition, backupItem.EndPosition) {+ // Extract statement text without trailing semicolon/whitespace.+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";")+ sqls = append(sqls, text)+ }+ default:}}++ return strings.Join(sqls, ";\n"), nil}func inRange(start, end, targetStart, targetEnd *storepb.Position) bool {@@ -316,110 +275,25 @@ func inRange(start, end, targetStart, targetEnd *storepb.Position) bool {}func getPrependStatements(statement string) (string, error) {- // Parse with ANTLR- parseResults, err := ParsePostgreSQL(statement)+ stmts, err := ParsePg(statement)if err != nil {return "", errors.Wrap(err, "failed to parse statement")}- // Create listener to find SET role statements across all parsed statements- listener := &setRoleListener{}- for _, parseResult := range parseResults {- antlr.ParseTreeWalkerDefault.Walk(listener, parseResult.Tree)- // If we found a SET role statement, return it immediately- if listener.setRoleText != "" {- break+ for _, stmt := range stmts {+ if stmt.Empty() {+ continue}- }-- return listener.setRoleText, nil-}--// setRoleListener detects SET role and search_path statements-type setRoleListener struct {- *parser.BasePostgreSQLParserListener- setRoleText string-}--// EnterVariablesetstmt handles SET statements-func (l *setRoleListener) EnterVariablesetstmt(ctx *parser.VariablesetstmtContext) {- // Only process if we haven't found a SET role/search_path statement yet- if l.setRoleText != "" {- return- }-- setRest := ctx.Set_rest()- if setRest == nil {- return- }- setRestMore := setRest.Set_rest_more()- if setRestMore == nil {- return- }-- // Check for "SET ROLE <name>" syntax (ROLE is a keyword/terminal)- // This is different from "SET role = <value>" which uses Generic_set- for i := 0; i < setRestMore.GetChildCount(); i++ {- child := setRestMore.GetChild(i)- if tn, ok := child.(*antlr.TerminalNodeImpl); ok {- if tn.GetText() == "ROLE" {- // Found SET ROLE syntax- l.setRoleText = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- return- }+ vs, ok := stmt.AST.(*ast.VariableSetStmt)+ if !ok {+ continue+ }+ name := strings.ToLower(vs.Name)+ if name == "role" || name == "search_path" {+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";")+ return text, nil}}- // Check for "SET role = <value>" or "SET search_path = <value>" syntax- // Structure: VariablesetstmtContext -> Set_rest -> Set_rest_more -> Generic_set -> Var_name- genericSet := setRestMore.Generic_set()- if genericSet == nil {- return- }- varName := genericSet.Var_name()- if varName == nil {- return- }- if len(varName.AllColid()) != 1 {- return- }-- name := NormalizePostgreSQLColid(varName.Colid(0))- if name == "role" || name == "search_path" {- // Found SET role or search_path statement, capture the full text- l.setRoleText = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)- }-}--// statementAtPositionFinder finds if a parse tree contains a statement at the given position.-type statementAtPositionFinder struct {- *parser.BasePostgreSQLParserListener- startPos *storepb.Position- endPos *storepb.Position- baseLine int- found bool-}--func (f *statementAtPositionFinder) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {- if isTopLevel(ctx.GetParent()) && inRange(&storepb.Position{- Line: int32(ctx.GetStart().GetLine()) + int32(f.baseLine),- Column: int32(ctx.GetStart().GetColumn()),- }, &storepb.Position{- Line: int32(ctx.GetStop().GetLine()) + int32(f.baseLine),- Column: int32(ctx.GetStop().GetColumn()),- }, f.startPos, f.endPos) {- f.found = true- }-}--func (f *statementAtPositionFinder) EnterDeletestmt(ctx *parser.DeletestmtContext) {- if isTopLevel(ctx.GetParent()) && inRange(&storepb.Position{- Line: int32(ctx.GetStart().GetLine()) + int32(f.baseLine),- Column: int32(ctx.GetStart().GetColumn()),- }, &storepb.Position{- Line: int32(ctx.GetStop().GetLine()) + int32(f.baseLine),- Column: int32(ctx.GetStop().GetColumn()),- }, f.startPos, f.endPos) {- f.found = true- }+ return "", nil}diff --git a/go.mod b/go.modindex 0e485a28f0..17eb8801cb 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c+ github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0edgithub.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex c6042468f0..87e75193d7 100644--- a/go.sum+++ b/go.sum@@ -253,8 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65 h1:RvViQx0e24DeDJgithub.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65/go.mod h1:4dDI0RO2BK9/BD4VE+YqvlCj/qoVMquZ99QZdRlT8Wg=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c h1:XHWlhrB4hVnDY4OY3pU0dwjHQJFkXuoWrzyDGjVpmiE=-github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c/go.mod h1:btZOCeNcueZehdqT+7YsLSVm+srVGSoPZYvFlITO2Z8=+github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0ed h1:vJ//BI7+pFRujbQTKBVAcZtAwoLSrWKjMVXITo6P97c=+github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0ed/go.mod h1:btZOCeNcueZehdqT+7YsLSVm+srVGSoPZYvFlITO2Z8=github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697 h1:/WgqQxX/gLsHS1WOx1rs5M/8kE9ecQAsgnQmlSr7p3c=github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=