Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(pg): migrate backup and restore from ANTLR to omni parser (#19692)

rebeliced11a47a995e8
diff --git a/backend/plugin/parser/pg/backup.go b/backend/plugin/parser/pg/backup.go
index f65d460e3e..aa263d16f5 100644
--- a/backend/plugin/parser/pg/backup.go
+++ b/backend/plugin/parser/pg/backup.go
@@ -6,10 +6,9 @@ import (
"slices"
"strings"
- "github.com/antlr4-go/antlr/v4"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/postgresql"
+ "github.com/bytebase/omni/pg/ast"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -53,9 +52,11 @@ func (t *TableReference) String() string {
type statementInfo struct {
offset int
statement string
- tree antlr.ParserRuleContext
+ node ast.Node
table *TableReference
- baseLine int
+ startPos *storepb.Position
+ endPos *storepb.Position
+ fullSQL string
}
func init() {
@@ -136,7 +137,7 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,
}
// In PostgreSQL, WITH clause must appear once before the first SELECT.
- cteClause := extractCTE(statementInfoList[0].tree)
+ cteClause := extractCTE(statementInfoList[0].node, statementInfoList[0].fullSQL)
if cteClause != "" {
if _, err := fmt.Fprintf(&buf, "%s\n", cteClause); err != nil {
return nil, errors.Wrap(err, "failed to write to buffer")
@@ -159,7 +160,7 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,
}
}
- if err := writeSuffixSelectClause(&buf, item.tree); err != nil {
+ if err := writeSuffixSelectClause(&buf, item.node, item.fullSQL); err != nil {
return nil, errors.Wrap(err, "failed to write string with new line")
}
}
@@ -173,98 +174,106 @@ func generateSQLForTable(statementInfoList []statementInfo, targetSchema string,
SourceSchema: table.Schema,
SourceTableName: table.Table,
TargetTableName: targetTable,
- StartPosition: &storepb.Position{
- Line: int32(statementInfoList[0].tree.GetStart().GetLine() + statementInfoList[0].baseLine),
- Column: int32(statementInfoList[0].tree.GetStart().GetColumn()),
- },
- EndPosition: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetLine()+statementInfoList[len(statementInfoList)-1].baseLine),
- int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetColumn()),
- statementInfoList[len(statementInfoList)-1].tree.GetStop().GetText(),
- ),
+ StartPosition: statementInfoList[0].startPos,
+ EndPosition: statementInfoList[len(statementInfoList)-1].endPos,
}, nil
}
-func extractCTE(ctx antlr.ParserRuleContext) string {
- switch node := ctx.(type) {
- case *parser.UpdatestmtContext:
- if optWith := node.Opt_with_clause(); optWith != nil {
- if withClause := optWith.With_clause(); withClause != nil {
- return node.GetParser().GetTokenStream().GetTextFromRuleContext(withClause)
- }
- }
- case *parser.DeletestmtContext:
- if optWith := node.Opt_with_clause(); optWith != nil {
- if withClause := optWith.With_clause(); withClause != nil {
- return node.GetParser().GetTokenStream().GetTextFromRuleContext(withClause)
- }
- }
+func extractCTE(node ast.Node, fullSQL string) string {
+ var wc *ast.WithClause
+ switch n := node.(type) {
+ case *ast.UpdateStmt:
+ wc = n.WithClause
+ case *ast.DeleteStmt:
+ wc = n.WithClause
default:
}
- return ""
-}
-
-func writeSuffixSelectClause(buf *strings.Builder, tree antlr.Tree) error {
- extractor := &suffixSelectClauseExtractor{
- buf: buf,
+ if wc == nil {
+ return ""
}
- antlr.ParseTreeWalkerDefault.Walk(extractor, tree)
- return extractor.err
+ return extractNodeText(wc.Loc, fullSQL)
}
-type suffixSelectClauseExtractor struct {
- *parser.BasePostgreSQLParserListener
-
- buf *strings.Builder
- err error
+func writeSuffixSelectClause(buf *strings.Builder, node ast.Node, fullSQL string) error {
+ switch n := node.(type) {
+ case *ast.UpdateStmt:
+ return writeUpdateSuffix(buf, n, fullSQL)
+ case *ast.DeleteStmt:
+ return writeDeleteSuffix(buf, n, fullSQL)
+ }
+ return nil
}
-func (e *suffixSelectClauseExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {
- if e.err != nil || !isTopLevel(ctx.GetParent()) {
- return
+func writeUpdateSuffix(buf *strings.Builder, n *ast.UpdateStmt, sql string) error {
+ if n.Relation == nil {
+ return nil
}
- if _, err := fmt.Fprintf(e.buf, "FROM %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Relation_expr_opt_alias())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ relText := extractNodeText(n.Relation.Loc, sql)
+ if _, err := fmt.Fprintf(buf, "FROM %s", relText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
}
- if ctx.From_clause() != nil {
- if _, err := fmt.Fprintf(e.buf, ", %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.From_clause().From_list())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ if n.FromClause != nil && n.FromClause.Len() > 0 {
+ fromText := extractNodeText(ast.ListSpan(n.FromClause), sql)
+ if fromText != "" {
+ if _, err := fmt.Fprintf(buf, ", %s", fromText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
+ }
}
}
- if ctx.Where_or_current_clause() != nil {
- if _, err := fmt.Fprintf(e.buf, " %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_or_current_clause())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ if n.WhereClause != nil {
+ whereText := extractNodeText(ast.NodeLoc(n.WhereClause), sql)
+ if whereText != "" {
+ if _, err := fmt.Fprintf(buf, " WHERE %s", whereText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
+ }
}
}
+ return nil
}
-func (e *suffixSelectClauseExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {
- if e.err != nil || !isTopLevel(ctx.GetParent()) {
- return
+func writeDeleteSuffix(buf *strings.Builder, n *ast.DeleteStmt, sql string) error {
+ if n.Relation == nil {
+ return nil
}
- if _, err := fmt.Fprintf(e.buf, "FROM %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Relation_expr_opt_alias())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ relText := extractNodeText(n.Relation.Loc, sql)
+ if _, err := fmt.Fprintf(buf, "FROM %s", relText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
}
- if ctx.Using_clause() != nil {
- if _, err := fmt.Fprintf(e.buf, ", %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Using_clause().From_list())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ if n.UsingClause != nil && n.UsingClause.Len() > 0 {
+ usingText := extractNodeText(ast.ListSpan(n.UsingClause), sql)
+ if usingText != "" {
+ if _, err := fmt.Fprintf(buf, ", %s", usingText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
+ }
}
}
- if ctx.Where_or_current_clause() != nil {
- if _, err := fmt.Fprintf(e.buf, " %s", ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_or_current_clause())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
+ if n.WhereClause != nil {
+ whereText := extractNodeText(ast.NodeLoc(n.WhereClause), sql)
+ if whereText != "" {
+ if _, err := fmt.Fprintf(buf, " WHERE %s", whereText); err != nil {
+ return errors.Wrap(err, "failed to write to buffer")
+ }
}
}
+ return nil
+}
+
+// extractNodeText extracts trimmed text from SQL using a Loc range.
+func extractNodeText(loc ast.Loc, sql string) string {
+ if loc.Start < 0 || loc.End < 0 || loc.Start >= loc.End || loc.End > len(sql) {
+ return ""
+ }
+ return strings.TrimSpace(sql[loc.Start:loc.End])
}
func prepareTransformation(ctx context.Context, tCtx base.TransformContext, statement string) ([]statementInfo, error) {
- parseResults, err := ParsePostgreSQL(statement)
+ stmts, err := ParsePg(statement)
if err != nil {
return nil, errors.Wrap(err, "failed to parse statement")
}
@@ -278,178 +287,111 @@ func prepareTransformation(ctx context.Context, tCtx base.TransformContext, stat
return nil, errors.Wrap(err, "failed to get database metadata")
}
- extractor := &dmlExtractor{
- metadata: metadata,
- searchPath: metadata.GetSearchPath(),
- }
+ searchPath := metadata.GetSearchPath()
+ var dmls []statementInfo
- // Walk all parse results to extract DML statements
- for _, parseResult := range parseResults {
- extractor.currentBaseLine = base.GetLineOffset(parseResult.StartPosition)
- antlr.ParseTreeWalkerDefault.Walk(extractor, parseResult.Tree)
- if extractor.err != nil {
- return nil, extractor.err
+ for i, stmt := range stmts {
+ if stmt.Empty() {
+ continue
}
- }
-
- return extractor.dmls, nil
-}
-type dmlExtractor struct {
- *parser.BasePostgreSQLParserListener
-
- metadata *model.DatabaseMetadata
- searchPath []string
- dmls []statementInfo
- offset int
- err error
- currentBaseLine int
-}
-
-func isTopLevel(ctx antlr.Tree) bool {
- if ctx == nil {
- return true
- }
-
- switch ctx := ctx.(type) {
- case *parser.RootContext, *parser.StmtblockContext:
- return true
- case *parser.StmtmultiContext, *parser.StmtContext:
- return isTopLevel(ctx.GetParent())
- default:
- return false
- }
-}
-
-func (e *dmlExtractor) EnterVariablesetstmt(ctx *parser.VariablesetstmtContext) {
- setRest := ctx.Set_rest()
- if setRest == nil {
- return
- }
- setRestMore := setRest.Set_rest_more()
- if setRestMore == nil {
- return
- }
- genericSet := setRestMore.Generic_set()
- if genericSet == nil {
- return
- }
- varName := genericSet.Var_name()
- if varName == nil {
- return
- }
- if len(varName.AllColid()) != 1 {
- return
- }
- name := NormalizePostgreSQLColid(varName.Colid(0))
- if !strings.EqualFold(name, "search_path") {
- return
- }
- var searchPath []string
- for _, value := range genericSet.Var_list().AllVar_value() {
- valueText := value.GetText()
- if strings.HasPrefix(valueText, "\"") && strings.HasSuffix(valueText, "\"") {
- // Remove the quotes from the schema name.
- valueText = strings.Trim(valueText, "\"")
- } else if strings.HasPrefix(valueText, "'") && strings.HasSuffix(valueText, "'") {
- // Remove the quotes from the schema name.
- valueText = strings.Trim(valueText, "'")
- } else {
- // For non-quoted schema names, we just return the lower string for PostgreSQL.
- valueText = strings.ToLower(valueText)
- }
- searchPath = append(searchPath, strings.TrimSpace(valueText))
- }
- e.searchPath = searchPath
-}
-
-func (e *dmlExtractor) ExitStmt(ctx *parser.StmtContext) {
- if isTopLevel(ctx) {
- e.offset++
- }
-}
-
-func (e *dmlExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {
- if isTopLevel(ctx.GetParent()) {
- table, err := e.extractTableReference(ctx.Relation_expr_opt_alias())
- if err != nil {
- e.err = errors.Wrapf(err, "failed to extract table reference from update statement at offset %d", e.offset)
- return
- }
- if table == nil {
- return
+ switch n := stmt.AST.(type) {
+ case *ast.VariableSetStmt:
+ if strings.EqualFold(n.Name, "search_path") && n.Args != nil {
+ var newSearchPath []string
+ for _, arg := range n.Args.Items {
+ if ac, ok := arg.(*ast.A_Const); ok {
+ if s, ok := ac.Val.(*ast.String); ok {
+ newSearchPath = append(newSearchPath, s.Str)
+ }
+ }
+ }
+ if len(newSearchPath) > 0 {
+ searchPath = newSearchPath
+ }
+ }
+ case *ast.UpdateStmt:
+ table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to extract table reference from update statement at offset %d", i)
+ }
+ if table == nil {
+ continue
+ }
+ table.StatementType = StatementTypeUpdate
+ nodeLoc := n.Loc
+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)
+ startPos.Column-- // 0-based column to match existing convention
+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)
+ dmls = append(dmls, statementInfo{
+ offset: i,
+ statement: stmt.Text,
+ node: n,
+ table: table,
+ startPos: startPos,
+ endPos: endPos,
+ fullSQL: statement,
+ })
+ case *ast.DeleteStmt:
+ table, err := extractTableReferenceFromRangeVar(n.Relation, metadata, searchPath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to extract table reference from delete statement at offset %d", i)
+ }
+ if table == nil {
+ continue
+ }
+ table.StatementType = StatementTypeDelete
+ nodeLoc := n.Loc
+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)
+ startPos.Column--
+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)
+ dmls = append(dmls, statementInfo{
+ offset: i,
+ statement: stmt.Text,
+ node: n,
+ table: table,
+ startPos: startPos,
+ endPos: endPos,
+ fullSQL: statement,
+ })
+ default:
}
- table.StatementType = StatementTypeUpdate
- e.dmls = append(e.dmls, statementInfo{
- offset: e.offset,
- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- tree: ctx,
- table: table,
- baseLine: e.currentBaseLine,
- })
}
-}
-func (e *dmlExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {
- if isTopLevel(ctx.GetParent()) {
- table, err := e.extractTableReference(ctx.Relation_expr_opt_alias())
- if err != nil {
- e.err = errors.Wrapf(err, "failed to extract table reference from delete statement at offset %d", e.offset)
- return
- }
- if table == nil {
- return
- }
- table.StatementType = StatementTypeDelete
- e.dmls = append(e.dmls, statementInfo{
- offset: e.offset,
- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- tree: ctx,
- table: table,
- baseLine: e.currentBaseLine,
- })
- }
+ return dmls, nil
}
-func (e *dmlExtractor) extractTableReference(ctx parser.IRelation_expr_opt_aliasContext) (*TableReference, error) {
- if ctx == nil {
+func extractTableReferenceFromRangeVar(rv *ast.RangeVar, metadata *model.DatabaseMetadata, searchPath []string) (*TableReference, error) {
+ if rv == nil {
return nil, nil
}
table := TableReference{}
- relationExpr := ctx.Relation_expr()
- if relationExpr == nil {
- return nil, nil
- }
-
- list := NormalizePostgreSQLQualifiedName(relationExpr.Qualified_name())
- switch len(list) {
- case 3:
- table.Database = list[0]
- table.Schema = list[1]
- table.Table = list[2]
- case 2:
- table.Schema = list[0]
- table.Table = list[1]
- case 1:
+ switch {
+ case rv.Catalogname != "":
+ table.Database = rv.Catalogname
+ table.Schema = rv.Schemaname
+ table.Table = rv.Relname
+ case rv.Schemaname != "":
+ table.Schema = rv.Schemaname
+ table.Table = rv.Relname
+ default:
// TODO: remove it in the future.
// Handle the case where the search path is not synchronized with the metadata.
- if len(e.searchPath) == 0 {
- e.searchPath = []string{"public"}
+ if len(searchPath) == 0 {
+ searchPath = []string{"public"}
}
- schemaName, _ := e.metadata.SearchObject(e.searchPath, list[0])
+ schemaName, _ := metadata.SearchObject(searchPath, rv.Relname)
if schemaName == "" {
- return nil, errors.Errorf("Table %q not found in metadata with search path %v", list[0], e.searchPath)
+ return nil, errors.Errorf("Table %q not found in metadata with search path %v", rv.Relname, searchPath)
}
table.Schema = schemaName
- table.Table = list[0]
- default:
- return nil, errors.Errorf("Invalid table name: %v", list)
+ table.Table = rv.Relname
}
- if ctx.Colid() != nil {
- table.Alias = NormalizePostgreSQLColid(ctx.Colid())
+ if rv.Alias != nil && rv.Alias.Aliasname != "" {
+ table.Alias = rv.Alias.Aliasname
}
return &table, nil
}
diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.go
index 24e1d56ba8..84af08edaf 100644
--- a/backend/plugin/parser/pg/pg.go
+++ b/backend/plugin/parser/pg/pg.go
@@ -281,27 +281,6 @@ func NormalizePostgreSQLQualifiedName(ctx parser.IQualified_nameContext) []strin
return res
}
-func normalizePostgreSQLSetTarget(ctx parser.ISet_targetContext) []string {
- if ctx == nil {
- return []string{}
- }
-
- var res []string
- res = append(res, NormalizePostgreSQLColid(ctx.Colid()))
- res = append(res, normalizePostgreSQLOptIndirection(ctx.Opt_indirection())...)
- return res
-}
-
-func normalizePostgreSQLOptIndirection(ctx parser.IOpt_indirectionContext) []string {
- var res []string
- for _, child := range ctx.AllIndirection_el() {
- if part := normalizePostgreSQLIndirectionEl(child); part != "" {
- res = append(res, part)
- }
- }
- return res
-}
-
func normalizePostgreSQLIndirection(ctx parser.IIndirectionContext) []string {
if ctx == nil {
return []string{}
diff --git a/backend/plugin/parser/pg/restore.go b/backend/plugin/parser/pg/restore.go
index 47fe06f3fa..02c1dd4039 100644
--- a/backend/plugin/parser/pg/restore.go
+++ b/backend/plugin/parser/pg/restore.go
@@ -5,10 +5,9 @@ import (
"fmt"
"strings"
- "github.com/antlr4-go/antlr/v4"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/postgresql"
+ "github.com/bytebase/omni/pg/ast"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -34,28 +33,12 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statement
return "", errors.Errorf("no original SQL")
}
- parseResults, err := ParsePostgreSQL(statement)
+ // Find the matching DML statement node at the backup position.
+ matchingNode, err := findStatementAtPosition(statement, backupItem)
if err != nil {
return "", err
}
-
- // Find the parse result that contains the statement at the backup position
- var targetResult *base.ANTLRAST
- for _, parseResult := range parseResults {
- // Walk the tree to find if this parse result contains the target statement
- finder := &statementAtPositionFinder{
- startPos: backupItem.StartPosition,
- endPos: backupItem.EndPosition,
- baseLine: base.GetLineOffset(parseResult.StartPosition),
- }
- antlr.ParseTreeWalkerDefault.Walk(finder, parseResult.Tree)
- if finder.found {
- targetResult = parseResult
- break
- }
- }
-
- if targetResult == nil {
+ if matchingNode == nil {
return "", errors.Errorf("could not find statement at position (line %d:%d - %d:%d)",
backupItem.StartPosition.Line, backupItem.StartPosition.Column,
backupItem.EndPosition.Line, backupItem.EndPosition.Column)
@@ -71,10 +54,35 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statement
return "", errors.Wrap(err, "failed to get prepend statements")
}
- return doGenerate(ctx, rCtx, sqlForComment, targetResult, backupItem, prependStatements)
+ return doGenerate(ctx, rCtx, sqlForComment, matchingNode, backupItem, prependStatements)
+}
+
+func findStatementAtPosition(statement string, backupItem *storepb.PriorBackupDetail_Item) (ast.Node, error) {
+ stmts, err := ParsePg(statement)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse statement")
+ }
+
+ for _, stmt := range stmts {
+ if stmt.Empty() {
+ continue
+ }
+ switch stmt.AST.(type) {
+ case *ast.UpdateStmt, *ast.DeleteStmt:
+ nodeLoc := ast.NodeLoc(stmt.AST)
+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)
+ startPos.Column-- // 0-based to match backup convention
+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)
+ if inRange(startPos, endPos, backupItem.StartPosition, backupItem.EndPosition) {
+ return stmt.AST, nil
+ }
+ default:
+ }
+ }
+ return nil, nil
}
-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, tree *base.ANTLRAST, backupItem *storepb.PriorBackupDetail_Item, prependStatements string) (string, error) {
+func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node ast.Node, backupItem *storepb.PriorBackupDetail_Item, prependStatements string) (string, error) {
_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)
if err != nil {
return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)
@@ -107,19 +115,42 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment str
return "", errors.Errorf("table metadata not found for %s.%s", schema, backupItem.SourceTable.Table)
}
- g := &generator{
- ctx: ctx,
- rCtx: rCtx,
- backupSchema: backupItem.TargetTable.Schema,
- backupTable: backupItem.TargetTable.Table,
- originalSchema: schema,
- originalTable: backupItem.SourceTable.Table,
- table: tableMetadata,
- isFirst: true,
- }
- antlr.ParseTreeWalkerDefault.Walk(g, tree.Tree)
- if g.err != nil {
- return "", g.err
+ backupSchema := backupItem.TargetTable.Schema
+ backupTable := backupItem.TargetTable.Table
+ originalSchema := schema
+ originalTable := backupItem.SourceTable.Table
+
+ var result string
+ switch n := node.(type) {
+ case *ast.DeleteStmt:
+ result = fmt.Sprintf(`INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s";`, originalSchema, originalTable, backupSchema, backupTable)
+ case *ast.UpdateStmt:
+ fields := extractSetFieldNames(n)
+ uk, err := findDisjointUniqueKey(tableMetadata, fields)
+ if err != nil {
+ return "", err
+ }
+
+ var buf strings.Builder
+ if _, err := fmt.Fprintf(&buf, `INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s" ON CONFLICT ON CONSTRAINT "%s" DO UPDATE SET `, originalSchema, originalTable, backupSchema, backupTable, uk); err != nil {
+ return "", errors.Wrapf(err, "failed to generate update statement")
+ }
+ for i, field := range fields {
+ if i > 0 {
+ if _, err := fmt.Fprint(&buf, ", "); err != nil {
+ return "", errors.Wrapf(err, "failed to generate update statement")
+ }
+ }
+ if _, err := fmt.Fprintf(&buf, `"%s" = EXCLUDED."%s"`, field, field); err != nil {
+ return "", errors.Wrapf(err, "failed to generate update statement")
+ }
+ }
+ if _, err := fmt.Fprint(&buf, `;`); err != nil {
+ return "", errors.Wrapf(err, "failed to generate update statement")
+ }
+ result = buf.String()
+ default:
+ return "", errors.Errorf("unexpected statement type: %T", node)
}
if len(prependStatements) > 0 {
@@ -127,32 +158,42 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment str
if !strings.HasSuffix(prependStatements, ";") {
prependStatements += ";"
}
- return fmt.Sprintf("%s\n/*\nOriginal SQL:\n%s\n*/\n%s", prependStatements, sqlForComment, g.result), nil
+ return fmt.Sprintf("%s\n/*\nOriginal SQL:\n%s\n*/\n%s", prependStatements, sqlForComment, result), nil
}
- return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result), nil
-}
-
-type generator struct {
- *parser.BasePostgreSQLParserListener
-
- backupSchema string
- backupTable string
- originalSchema string
- originalTable string
- table *model.TableMetadata
-
- isFirst bool
- ctx context.Context
- rCtx base.RestoreContext
- result string
- err error
+ return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, result), nil
}
-func (g *generator) EnterDeletestmt(ctx *parser.DeletestmtContext) {
- if isTopLevel(ctx.GetParent()) && g.isFirst {
- g.isFirst = false
- g.result = fmt.Sprintf(`INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s";`, g.originalSchema, g.originalTable, g.backupSchema, g.backupTable)
+// extractSetFieldNames extracts the target column names from an UPDATE SET clause.
+func extractSetFieldNames(stmt *ast.UpdateStmt) []string {
+ if stmt.TargetList == nil {
+ return nil
+ }
+ var fields []string
+ for _, item := range stmt.TargetList.Items {
+ rt, ok := item.(*ast.ResTarget)
+ if !ok || rt.Name == "" {
+ continue
+ }
+ // Take the last name component (the actual column name).
+ // For "SET test.c1 = 1", Name="test", Indirection=["c1"] → want "c1".
+ // For "SET c1 = 1", Name="c1", Indirection=nil → want "c1".
+ // For "SET schema.col[1] = val", Indirection=["col", integer] → want "col".
+ if rt.Indirection != nil && rt.Indirection.Len() > 0 {
+ found := false
+ for j := rt.Indirection.Len() - 1; j >= 0; j-- {
+ if s, ok := rt.Indirection.Items[j].(*ast.String); ok {
+ fields = append(fields, s.Str)
+ found = true
+ break
+ }
+ }
+ if found {
+ continue
+ }
+ }
+ fields = append(fields, rt.Name)
}
+ return fields
}
func disjoint(a []string, b map[string]bool) bool {
@@ -164,18 +205,18 @@ func disjoint(a []string, b map[string]bool) bool {
return true
}
-func (g *generator) findDisjointUniqueKey(fields []string) (string, error) {
+func findDisjointUniqueKey(table *model.TableMetadata, fields []string) (string, error) {
columnMap := make(map[string]bool)
for _, field := range fields {
columnMap[field] = true
}
- pk := g.table.GetPrimaryKey()
+ pk := table.GetPrimaryKey()
if pk != nil {
if disjoint(pk.GetProto().Expressions, columnMap) {
return pk.GetProto().Name, nil
}
}
- for _, index := range g.table.GetProto().Indexes {
+ for _, index := range table.GetProto().Indexes {
if index.Primary {
continue
}
@@ -187,61 +228,7 @@ func (g *generator) findDisjointUniqueKey(fields []string) (string, error) {
}
}
- return "", errors.Errorf("no disjoint unique key found for %s.%s", g.originalSchema, g.originalTable)
-}
-
-func (g *generator) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {
- if isTopLevel(ctx.GetParent()) && g.isFirst {
- g.isFirst = false
-
- l := &setFieldListener{}
- antlr.ParseTreeWalkerDefault.Walk(l, ctx)
-
- uk, err := g.findDisjointUniqueKey(l.result)
- if err != nil {
- g.err = err
- return
- }
-
- var buf strings.Builder
- if _, err := fmt.Fprintf(&buf, `INSERT INTO "%s"."%s" SELECT * FROM "%s"."%s" ON CONFLICT ON CONSTRAINT "%s" DO UPDATE SET `, g.originalSchema, g.originalTable, g.backupSchema, g.backupTable, uk); err != nil {
- g.err = errors.Wrapf(err, "failed to generate update statement")
- return
- }
- for i, field := range l.result {
- if i > 0 {
- if _, err := fmt.Fprint(&buf, ", "); err != nil {
- g.err = errors.Wrapf(err, "failed to generate update statement")
- return
- }
- }
- // The field is written by user and no need to escape.
- if _, err := fmt.Fprintf(&buf, `"%s" = EXCLUDED."%s"`, field, field); err != nil {
- g.err = errors.Wrapf(err, "failed to generate update statement")
- return
- }
- }
- if _, err := fmt.Fprint(&buf, `;`); err != nil {
- g.err = errors.Wrapf(err, "failed to generate update statement")
- return
- }
- g.result = buf.String()
- }
-}
-
-type setFieldListener struct {
- *parser.BasePostgreSQLParserListener
-
- result []string
-}
-
-func (l *setFieldListener) EnterSet_clause(ctx *parser.Set_clauseContext) {
- if ctx.Set_target() != nil {
- names := normalizePostgreSQLSetTarget(ctx.Set_target())
- if len(names) > 0 {
- l.result = append(l.result, names[len(names)-1])
- }
- }
+ return "", errors.Errorf("no disjoint unique key found for table")
}
func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
@@ -249,60 +236,32 @@ func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_It
return "", errors.Errorf("backup item is nil")
}
- parseResults, err := ParsePostgreSQL(statement)
+ stmts, err := ParsePg(statement)
if err != nil {
return "", errors.Wrapf(err, "failed to parse statement")
}
- l := &originalSQLExtractor{
- startPos: backupItem.StartPosition,
- endPos: backupItem.EndPosition,
- }
-
- // Walk all parse results to find statements within the specified position range
- for _, parseResult := range parseResults {
- l.baseLine = base.GetLineOffset(parseResult.StartPosition)
- antlr.ParseTreeWalkerDefault.Walk(l, parseResult.Tree)
- }
-
- return strings.Join(l.originalSQL, ";\n"), nil
-}
-
-type originalSQLExtractor struct {
- *parser.BasePostgreSQLParserListener
-
- originalSQL []string
- startPos *storepb.Position
- endPos *storepb.Position
- baseLine int
-}
-
-func (l *originalSQLExtractor) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {
- if isTopLevel(ctx.GetParent()) {
- if inRange(&storepb.Position{
- Line: int32(ctx.GetStart().GetLine()) + int32(l.baseLine),
- Column: int32(ctx.GetStart().GetColumn()),
- }, &storepb.Position{
- Line: int32(ctx.GetStop().GetLine()) + int32(l.baseLine),
- Column: int32(ctx.GetStop().GetColumn()),
- }, l.startPos, l.endPos) {
- l.originalSQL = append(l.originalSQL, ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx))
+ var sqls []string
+ for _, stmt := range stmts {
+ if stmt.Empty() {
+ continue
}
- }
-}
-
-func (l *originalSQLExtractor) EnterDeletestmt(ctx *parser.DeletestmtContext) {
- if isTopLevel(ctx.GetParent()) {
- if inRange(&storepb.Position{
- Line: int32(ctx.GetStart().GetLine()) + int32(l.baseLine),
- Column: int32(ctx.GetStart().GetColumn()),
- }, &storepb.Position{
- Line: int32(ctx.GetStop().GetLine()) + int32(l.baseLine),
- Column: int32(ctx.GetStop().GetColumn()),
- }, l.startPos, l.endPos) {
- l.originalSQL = append(l.originalSQL, ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx))
+ switch stmt.AST.(type) {
+ case *ast.UpdateStmt, *ast.DeleteStmt:
+ nodeLoc := ast.NodeLoc(stmt.AST)
+ startPos := ByteOffsetToRunePosition(statement, nodeLoc.Start)
+ startPos.Column-- // 0-based
+ endPos := ByteOffsetToRunePosition(statement, nodeLoc.End)
+ if inRange(startPos, endPos, backupItem.StartPosition, backupItem.EndPosition) {
+ // Extract statement text without trailing semicolon/whitespace.
+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";")
+ sqls = append(sqls, text)
+ }
+ default:
}
}
+
+ return strings.Join(sqls, ";\n"), nil
}
func inRange(start, end, targetStart, targetEnd *storepb.Position) bool {
@@ -316,110 +275,25 @@ func inRange(start, end, targetStart, targetEnd *storepb.Position) bool {
}
func getPrependStatements(statement string) (string, error) {
- // Parse with ANTLR
- parseResults, err := ParsePostgreSQL(statement)
+ stmts, err := ParsePg(statement)
if err != nil {
return "", errors.Wrap(err, "failed to parse statement")
}
- // Create listener to find SET role statements across all parsed statements
- listener := &setRoleListener{}
- for _, parseResult := range parseResults {
- antlr.ParseTreeWalkerDefault.Walk(listener, parseResult.Tree)
- // If we found a SET role statement, return it immediately
- if listener.setRoleText != "" {
- break
+ for _, stmt := range stmts {
+ if stmt.Empty() {
+ continue
}
- }
-
- return listener.setRoleText, nil
-}
-
-// setRoleListener detects SET role and search_path statements
-type setRoleListener struct {
- *parser.BasePostgreSQLParserListener
- setRoleText string
-}
-
-// EnterVariablesetstmt handles SET statements
-func (l *setRoleListener) EnterVariablesetstmt(ctx *parser.VariablesetstmtContext) {
- // Only process if we haven't found a SET role/search_path statement yet
- if l.setRoleText != "" {
- return
- }
-
- setRest := ctx.Set_rest()
- if setRest == nil {
- return
- }
- setRestMore := setRest.Set_rest_more()
- if setRestMore == nil {
- return
- }
-
- // Check for "SET ROLE <name>" syntax (ROLE is a keyword/terminal)
- // This is different from "SET role = <value>" which uses Generic_set
- for i := 0; i < setRestMore.GetChildCount(); i++ {
- child := setRestMore.GetChild(i)
- if tn, ok := child.(*antlr.TerminalNodeImpl); ok {
- if tn.GetText() == "ROLE" {
- // Found SET ROLE syntax
- l.setRoleText = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- return
- }
+ vs, ok := stmt.AST.(*ast.VariableSetStmt)
+ if !ok {
+ continue
+ }
+ name := strings.ToLower(vs.Name)
+ if name == "role" || name == "search_path" {
+ text := strings.TrimRight(strings.TrimSpace(stmt.Text), ";")
+ return text, nil
}
}
- // Check for "SET role = <value>" or "SET search_path = <value>" syntax
- // Structure: VariablesetstmtContext -> Set_rest -> Set_rest_more -> Generic_set -> Var_name
- genericSet := setRestMore.Generic_set()
- if genericSet == nil {
- return
- }
- varName := genericSet.Var_name()
- if varName == nil {
- return
- }
- if len(varName.AllColid()) != 1 {
- return
- }
-
- name := NormalizePostgreSQLColid(varName.Colid(0))
- if name == "role" || name == "search_path" {
- // Found SET role or search_path statement, capture the full text
- l.setRoleText = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
- }
-}
-
-// statementAtPositionFinder finds if a parse tree contains a statement at the given position.
-type statementAtPositionFinder struct {
- *parser.BasePostgreSQLParserListener
- startPos *storepb.Position
- endPos *storepb.Position
- baseLine int
- found bool
-}
-
-func (f *statementAtPositionFinder) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {
- if isTopLevel(ctx.GetParent()) && inRange(&storepb.Position{
- Line: int32(ctx.GetStart().GetLine()) + int32(f.baseLine),
- Column: int32(ctx.GetStart().GetColumn()),
- }, &storepb.Position{
- Line: int32(ctx.GetStop().GetLine()) + int32(f.baseLine),
- Column: int32(ctx.GetStop().GetColumn()),
- }, f.startPos, f.endPos) {
- f.found = true
- }
-}
-
-func (f *statementAtPositionFinder) EnterDeletestmt(ctx *parser.DeletestmtContext) {
- if isTopLevel(ctx.GetParent()) && inRange(&storepb.Position{
- Line: int32(ctx.GetStart().GetLine()) + int32(f.baseLine),
- Column: int32(ctx.GetStart().GetColumn()),
- }, &storepb.Position{
- Line: int32(ctx.GetStop().GetLine()) + int32(f.baseLine),
- Column: int32(ctx.GetStop().GetColumn()),
- }, f.startPos, f.endPos) {
- f.found = true
- }
+ return "", nil
}
diff --git a/go.mod b/go.mod
index 0e485a28f0..17eb8801cb 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
- github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c
+ github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0ed
github.com/caarlos0/env/v11 v11.4.0
github.com/cenkalti/backoff/v5 v5.0.3
github.com/cockroachdb/cockroachdb-parser v0.25.2
diff --git a/go.sum b/go.sum
index c6042468f0..87e75193d7 100644
--- a/go.sum
+++ b/go.sum
@@ -253,8 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65 h1:RvViQx0e24DeDJ
github.com/bytebase/gomongo v0.0.0-20260211083148-d2ea344b4b65/go.mod h1:4dDI0RO2BK9/BD4VE+YqvlCj/qoVMquZ99QZdRlT8Wg=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
-github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c h1:XHWlhrB4hVnDY4OY3pU0dwjHQJFkXuoWrzyDGjVpmiE=
-github.com/bytebase/omni v0.0.0-20260320111558-ef957ff3b59c/go.mod h1:btZOCeNcueZehdqT+7YsLSVm+srVGSoPZYvFlITO2Z8=
+github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0ed h1:vJ//BI7+pFRujbQTKBVAcZtAwoLSrWKjMVXITo6P97c=
+github.com/bytebase/omni v0.0.0-20260323084215-7bbe88e0d0ed/go.mod h1:btZOCeNcueZehdqT+7YsLSVm+srVGSoPZYvFlITO2Z8=
github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697 h1:/WgqQxX/gLsHS1WOx1rs5M/8kE9ecQAsgnQmlSr7p3c=
github.com/bytebase/parser v0.0.0-20260318084752-6c474df57697/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=