Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/plsql): migrate rollback paths to omni (#20451)
rebelice2d27eceb1a11
diff --git a/backend/plugin/parser/plsql/backup.go b/backend/plugin/parser/plsql/backup.goindex c88bd46bb3..b0a057477b 100644--- a/backend/plugin/parser/plsql/backup.go+++ b/backend/plugin/parser/plsql/backup.go@@ -3,14 +3,13 @@ package plsqlimport ("context""fmt"+ "reflect""slices""strings"- "github.com/antlr4-go/antlr/v4"+ oracleast "github.com/bytebase/omni/oracle/ast""github.com/pkg/errors"- parser "github.com/bytebase/parser/plsql"-"github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -21,6 +20,8 @@ const (maxTableNameLengthBefore12_2 = 30)+var errNoBackupableDML = errors.New("no parse results")+func init() {base.RegisterTransformDMLToSelect(store.Engine_ORACLE, TransformDMLToSelect)}@@ -44,11 +45,13 @@ type TableReference struct {}type statementInfo struct {- offset int- statement string- tree antlr.ParserRuleContext- table *TableReference- baseLine int+ offset int+ statement string+ node oracleast.StmtNode+ table *TableReference+ startPosition *store.Position+ endPosition *store.Position+ fullSQL string}// TransformDMLToSelect transforms DML statement to SELECT statement.@@ -160,7 +163,7 @@ func generateSQLForTable(ctx base.TransformContext, statementInfoList []statemen}}}- if err := writeSuffixSelectClause(&buf, info.tree); err != nil {+ if err := writeSuffixSelectClause(&buf, info.node, info.fullSQL); err != nil {return nil, errors.Wrap(err, "failed to write suffix select clause")}}@@ -174,188 +177,240 @@ func generateSQLForTable(ctx base.TransformContext, statementInfoList []statemenSourceSchema: table.Schema,SourceTableName: table.Table,TargetTableName: targetTable,- StartPosition: &store.Position{- Line: int32(statementInfoList[0].tree.GetStart().GetLine() + statementInfoList[0].baseLine),- Column: int32(statementInfoList[0].tree.GetStart().GetColumn()),- },- EndPosition: &store.Position{- Line: int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetLine() + statementInfoList[len(statementInfoList)-1].baseLine),- Column: int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetColumn()),- },+ StartPosition: zeroBasedColumnPosition(statementInfoList[0].startPosition),+ EndPosition: zeroBasedColumnPosition(statementInfoList[len(statementInfoList)-1].endPosition),}, nil}-func writeSuffixSelectClause(buf *strings.Builder, tree antlr.Tree) error {- extractor := &suffixSelectClauseExtractor{- buf: buf,+func prepareTransformation(databaseName, statement string) ([]statementInfo, error) {+ statements, err := SplitSQL(statement)+ if err != nil {+ return nil, errors.Wrap(err, "failed to split PLSQL")}- antlr.ParseTreeWalkerDefault.Walk(extractor, tree)- return extractor.err-}--type suffixSelectClauseExtractor struct {- *parser.BasePlSqlParserListener-- buf *strings.Builder- err error-}-func (e *suffixSelectClauseExtractor) EnterDelete_statement(ctx *parser.Delete_statementContext) {- if e.err != nil || !IsTopLevelStatement(ctx.GetParent()) {- return+ var result []statementInfo+ positionMapper := base.NewByteOffsetPositionMapper(statement)+ for i, stmt := range statements {+ if stmt.Empty {+ continue+ }+ list, err := ParsePLSQLOmni(stmt.Text)+ if err != nil {+ return nil, errors.Wrap(err, "failed to parse PLSQL")+ }+ for _, item := range list.Items {+ raw, ok := item.(*oracleast.RawStmt)+ if !ok || raw.Stmt == nil {+ continue+ }+ info := extractOmniDML(databaseName, raw.Stmt, stmt.Text)+ if info == nil {+ continue+ }+ info.offset = i+ if stmt.Range != nil {+ info.startPosition = positionMapper.Position(int(stmt.Range.Start) + raw.Loc.Start)+ info.endPosition = positionMapper.Position(int(stmt.Range.Start) + max(raw.Loc.End-1, raw.Loc.Start))+ } else {+ info.startPosition = stmt.Start+ info.endPosition = stmt.End+ }+ info.fullSQL = stmt.Text+ result = append(result, *info)+ }}- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.General_table_ref())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+ if len(result) == 0 {+ return nil, errNoBackupableDML}+ return result, nil+}- if ctx.Where_clause() != nil {- if _, err := e.buf.WriteString(" "); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+func extractOmniDML(databaseName string, node oracleast.StmtNode, fullSQL string) *statementInfo {+ switch n := node.(type) {+ case *oracleast.DeleteStmt:+ table := omniDMLTableReference(databaseName, n.Table, n.Alias, StatementTypeDelete)+ if table == nil {+ return nil+ }+ return &statementInfo{+ statement: extractOmniStatementText(fullSQL, n.Loc),+ node: n,+ table: table,+ }+ case *oracleast.UpdateStmt:+ table := omniDMLTableReference(databaseName, n.Table, n.Alias, StatementTypeUpdate)+ if table == nil {+ return nil}- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+ return &statementInfo{+ statement: extractOmniStatementText(fullSQL, n.Loc),+ node: n,+ table: table,}+ default:+ return nil}}-func (e *suffixSelectClauseExtractor) EnterUpdate_statement(ctx *parser.Update_statementContext) {- if e.err != nil || !IsTopLevelStatement(ctx.GetParent()) {- return+func omniDMLTableReference(databaseName string, name *oracleast.ObjectName, alias *oracleast.Alias, statementType StatementType) *TableReference {+ if name == nil || name.Name == "" {+ return nil}-- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.General_table_ref())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+ schema := name.Schema+ hasSchema := schema != ""+ if schema == "" {+ schema = databaseName+ }+ table := &TableReference{+ Database: schema,+ HasSchema: hasSchema,+ Schema: schema,+ Table: name.Name,+ StatementType: statementType,+ }+ if alias != nil {+ table.Alias = alias.Name}+ return table+}- if ctx.Where_clause() != nil {- if _, err := e.buf.WriteString(" "); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+func writeSuffixSelectClause(buf *strings.Builder, node oracleast.StmtNode, fullSQL string) error {+ suffix := ""+ switch n := node.(type) {+ case *oracleast.UpdateStmt:+ suffix = oracleDMLTargetText(fullSQL, n.Table, n.PartitionExt, n.Alias)+ if n.WhereClause != nil && n.SetClauses != nil && n.SetClauses.Len() > 0 {+ if setClause, ok := n.SetClauses.Items[n.SetClauses.Len()-1].(*oracleast.SetClause); ok {+ whereStart := setClause.Loc.End+ if fromText, fromEnd, ok := oracleListText(fullSQL, n.FromClause); ok {+ suffix += ", " + fromText+ whereStart = fromEnd+ }+ suffix = joinOracleSuffix(suffix, oracleWhereSuffix(fullSQL, whereStart, n.WhereClause))+ }+ }+ case *oracleast.DeleteStmt:+ targetEnd := n.Table.Loc.End+ if n.PartitionExt != nil {+ targetEnd = n.PartitionExt.Loc.End}- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())); err != nil {- e.err = errors.Wrap(err, "failed to write to buffer")- return+ if n.Alias != nil {+ targetEnd = n.Alias.Loc.End}+ suffix = joinOracleSuffix(oracleDMLTargetText(fullSQL, n.Table, n.PartitionExt, n.Alias), oracleWhereSuffix(fullSQL, targetEnd, n.WhereClause))+ default:}+ _, err := buf.WriteString(suffix)+ return err}-func prepareTransformation(databaseName, statement string) ([]statementInfo, error) {- results, err := ParsePLSQL(statement)- if err != nil {- return nil, errors.Wrap(err, "failed to parse PLSQL")+func oracleDMLTargetText(sql string, name *oracleast.ObjectName, partitionExt *oracleast.PartitionExtClause, alias *oracleast.Alias) string {+ if name == nil {+ return ""}- if len(results) == 0 {- return nil, errors.New("no parse results")+ end := name.Loc.End+ if partitionExt != nil {+ end = partitionExt.Loc.End}-- extractor := &dmlExtractor{- databaseName: databaseName,- }-- // Walk each ANTLRAST tree to extract DML statements- for _, result := range results {- extractor.baseLine = base.GetLineOffset(result.StartPosition)- antlr.ParseTreeWalkerDefault.Walk(extractor, result.Tree)+ if alias != nil {+ end = alias.Loc.End}+ return strings.TrimSpace(oracleLocText(sql, name.Loc.Start, end))+}- return extractor.dmls, nil+func oracleDMLTrailingText(sql string, start, end int) string {+ text := strings.TrimSpace(oracleLocText(sql, start, end))+ return strings.TrimSpace(strings.TrimSuffix(text, ";"))}-func IsTopLevelStatement(ctx antlr.Tree) bool {- if ctx == nil {- return true+func oracleListText(sql string, list *oracleast.List) (string, int, bool) {+ if list == nil || list.Len() == 0 {+ return "", 0, false}- switch ctx := ctx.(type) {- case *parser.Unit_statementContext, *parser.Sql_scriptContext:- return true- case *parser.Data_manipulation_language_statementsContext:- return IsTopLevelStatement(ctx.GetParent())- default:- return false+ start, ok := oracleNodeLoc(list.Items[0])+ if !ok {+ return "", 0, false}+ end, ok := oracleNodeLoc(list.Items[list.Len()-1])+ if !ok {+ return "", 0, false+ }+ text := strings.TrimSpace(oracleLocText(sql, start.Start, end.End))+ if text == "" {+ return "", 0, false+ }+ return text, end.End, true}-type dmlExtractor struct {- *parser.BasePlSqlParserListener-- databaseName string- dmls []statementInfo- offset int- baseLine int-}--func (e *dmlExtractor) ExitUnit_statement(_ *parser.Unit_statementContext) {- e.offset+++func oracleWhereSuffix(sql string, start int, where oracleast.ExprNode) string {+ loc, ok := oracleNodeLoc(where)+ if !ok {+ return ""+ }+ return oracleDMLTrailingText(sql, start, loc.End)}-func (e *dmlExtractor) ExitSql_plus_command(_ *parser.Sql_plus_commandContext) {- e.offset+++func oracleNodeLoc(node oracleast.Node) (oracleast.Loc, bool) {+ if node == nil {+ return oracleast.Loc{}, false+ }+ value := reflect.ValueOf(node)+ if value.Kind() != reflect.Pointer || value.IsNil() {+ return oracleast.Loc{}, false+ }+ elem := value.Elem()+ if elem.Kind() != reflect.Struct {+ return oracleast.Loc{}, false+ }+ field := elem.FieldByName("Loc")+ if !field.IsValid() || field.Type() != reflect.TypeOf(oracleast.Loc{}) {+ return oracleast.Loc{}, false+ }+ loc, ok := field.Interface().(oracleast.Loc)+ if !ok || loc.End <= loc.Start {+ return oracleast.Loc{}, false+ }+ return loc, true}-func (e *dmlExtractor) EnterDelete_statement(ctx *parser.Delete_statementContext) {- if IsTopLevelStatement(ctx.GetParent()) {- extractor := &tableExtractor{- databaseName: e.databaseName,- }- antlr.ParseTreeWalkerDefault.Walk(extractor, ctx)- extractor.table.StatementType = StatementTypeDelete-- e.dmls = append(e.dmls, statementInfo{- offset: e.offset,- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- tree: ctx,- table: extractor.table,- baseLine: e.baseLine,- })+func oracleLocText(sql string, start, end int) string {+ if start < 0 {+ start = 0+ }+ if end <= 0 || end > len(sql) {+ end = len(sql)+ }+ if start >= end || start >= len(sql) {+ return ""}+ return sql[start:end]}-func (e *dmlExtractor) EnterUpdate_statement(ctx *parser.Update_statementContext) {- if IsTopLevelStatement(ctx.GetParent()) {- extractor := &tableExtractor{- databaseName: e.databaseName,+func joinOracleSuffix(parts ...string) string {+ var nonEmpty []string+ for _, part := range parts {+ if part != "" {+ nonEmpty = append(nonEmpty, part)}- antlr.ParseTreeWalkerDefault.Walk(extractor, ctx)- extractor.table.StatementType = StatementTypeUpdate-- e.dmls = append(e.dmls, statementInfo{- offset: e.offset,- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- tree: ctx,- table: extractor.table,- baseLine: e.baseLine,- })}+ return strings.Join(nonEmpty, " ")}-type tableExtractor struct {- *parser.BasePlSqlParserListener-- databaseName string- table *TableReference+func extractOmniStatementText(sql string, loc oracleast.Loc) string {+ return strings.TrimSpace(strings.TrimSuffix(oracleLocText(sql, loc.Start, loc.End), ";"))}-func (e *tableExtractor) EnterGeneral_table_ref(ctx *parser.General_table_refContext) {- dmlTableExpr := ctx.Dml_table_expression_clause()- if dmlTableExpr != nil && dmlTableExpr.Tableview_name() != nil {- _, schemaName, tableName := NormalizeTableViewName("", dmlTableExpr.Tableview_name())- e.table = &TableReference{- Database: schemaName,- HasSchema: true,- Schema: schemaName,- Table: tableName,- }- if schemaName == "" {- e.table.Schema = e.databaseName- e.table.HasSchema = false- }- if ctx.Table_alias() != nil {- e.table.Alias = NormalizeTableAlias(ctx.Table_alias())- }+func zeroBasedColumnPosition(pos *store.Position) *store.Position {+ if pos == nil {+ return nil+ }+ column := pos.Column+ if column > 0 {+ column--+ }+ return &store.Position{+ Line: pos.Line,+ Column: column,}}diff --git a/backend/plugin/parser/plsql/backup_test.go b/backend/plugin/parser/plsql/backup_test.goindex b18be8688d..3c1876ced4 100644--- a/backend/plugin/parser/plsql/backup_test.go+++ b/backend/plugin/parser/plsql/backup_test.go@@ -70,3 +70,81 @@ func TestBackup(t *testing.T) {yamltest.Record(t, filepath, tests)}}++func TestBackupOmniBoundaryCases(t *testing.T) {+ tests := []struct {+ name string+ input string+ wantSQL string+ }{+ {+ name: "update returning is not copied into select suffix",+ input: "UPDATE test SET c1 = 1 WHERE c1 = 2 RETURNING c1 INTO :old_c1;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test WHERE c1 = 2;`,+ },+ {+ name: "delete returning is not copied into select suffix",+ input: "DELETE FROM test WHERE c1 = 2 RETURNING c1 INTO :old_c1;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test WHERE c1 = 2;`,+ },+ {+ name: "update log errors is not copied into select suffix",+ input: "UPDATE test SET c1 = 1 WHERE c1 = 2 LOG ERRORS INTO err$_test REJECT LIMIT UNLIMITED;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test WHERE c1 = 2;`,+ },+ {+ name: "where keyword in block comment",+ input: "UPDATE test SET c1 = 1 WHERE /* WHERE */ c1 = 2;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test WHERE /* WHERE */ c1 = 2;`,+ },+ {+ name: "where keyword in string literal",+ input: "DELETE FROM test WHERE note = 'WHERE should stay literal';",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test WHERE note = 'WHERE should stay literal';`,+ },+ {+ name: "update without where backs up whole table",+ input: "UPDATE test SET c1 = 1;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test;`,+ },+ {+ name: "delete without where backs up whole table",+ input: "DELETE FROM test;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test;`,+ },+ {+ name: "update partition target",+ input: "UPDATE test PARTITION (p1) SET c1 = 1 WHERE c1 = 2;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test PARTITION (p1) WHERE c1 = 2;`,+ },+ {+ name: "delete subpartition target",+ input: "DELETE FROM test SUBPARTITION (sp1) WHERE c1 = 2;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "TEST".* FROM test SUBPARTITION (sp1) WHERE c1 = 2;`,+ },+ {+ name: "update from appends source table list",+ input: "UPDATE test t SET c1 = s.c1 FROM source_table s WHERE t.id = s.id;",+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS+ SELECT "T".* FROM test t, source_table s WHERE t.id = s.id;`,+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{}, tc.input, "DB", "backupDB", "rollback")+ require.NoError(t, err)+ require.Len(t, result, 1)+ require.Equal(t, tc.wantSQL, result[0].Statement)+ })+ }+}diff --git a/backend/plugin/parser/plsql/diagnose.go b/backend/plugin/parser/plsql/diagnose.goindex 5ddef63c5b..4fae75715b 100644--- a/backend/plugin/parser/plsql/diagnose.go+++ b/backend/plugin/parser/plsql/diagnose.go@@ -2,12 +2,10 @@ package plsqlimport ("context"+ "errors""strings""unicode"- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/plsql"-"github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -26,7 +24,6 @@ func Diagnose(_ context.Context, _ base.DiagnoseContext, statement string) ([]bareturn diagnostics, nil}-// ParsePLSQL parses the given PLSQL.func parsePLSQLStatement(statement string) *base.SyntaxError {trimmedStatement := strings.TrimRightFunc(statement, unicode.IsSpace)if len(trimmedStatement) > 0 && !strings.HasSuffix(trimmedStatement, ";") {@@ -35,32 +32,18 @@ func parsePLSQLStatement(statement string) *base.SyntaxError {statement += ";"}- lexer := parser.NewPlSqlLexer(antlr.NewInputStream(statement))- stream := antlr.NewCommonTokenStream(lexer, 0)- p := parser.NewPlSqlParser(stream)- p.SetVersion12(true)-- lexerErrorListener := &base.ParseErrorListener{- Statement: statement,- }- lexer.RemoveErrorListeners()- lexer.AddErrorListener(lexerErrorListener)-- parserErrorListener := &base.ParseErrorListener{- Statement: statement,- }- p.RemoveErrorListeners()- p.AddErrorListener(parserErrorListener)-- p.BuildParseTrees = false- _ = p.Sql_script()-- if lexerErrorListener.Err != nil {- return lexerErrorListener.Err- }-- if parserErrorListener.Err != nil {- return parserErrorListener.Err+ if _, err := ParsePLSQLOmni(statement); err != nil {+ if _, fallbackErr := ParsePLSQL(statement); fallbackErr == nil {+ return nil+ }+ var syntaxErr *base.SyntaxError+ if errors.As(convertOmniError(err, base.Statement{Text: statement}), &syntaxErr) {+ return syntaxErr+ }+ return &base.SyntaxError{+ Position: &store.Position{Line: 1, Column: 1},+ Message: err.Error(),+ }}return nil}diff --git a/backend/plugin/parser/plsql/diagnose_test.go b/backend/plugin/parser/plsql/diagnose_test.gonew file mode 100644index 0000000000..06ef431383--- /dev/null+++ b/backend/plugin/parser/plsql/diagnose_test.go@@ -0,0 +1,58 @@+package plsql++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestDiagnoseUsesOmniParseErrorPosition(t *testing.T) {+ tests := []struct {+ name string+ statement string+ line uint32+ }{+ {+ name: "single line",+ statement: "SELECT * FROM",+ line: 0,+ },+ {+ name: "second sql statement",+ statement: "SELECT 1 FROM DUAL;\n\nSELECT * FROM",+ line: 2,+ },+ {+ name: "line after non-ascii",+ statement: "SELECT '你好' FROM DUAL;\nSELECT * FROM",+ line: 1,+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ diagnostics, err := Diagnose(context.Background(), base.DiagnoseContext{}, tc.statement)+ require.NoError(t, err)+ require.Len(t, diagnostics, 1)+ require.Equal(t, tc.line, diagnostics[0].Range.Start.Line)+ })+ }+}++func TestDiagnoseFallsBackToANTLR(t *testing.T) {+ statement := `CREATE OR REPLACE TRIGGER trg+BEFORE INSERT OR UPDATE OF col1, col2 ON tbl+REFERENCING OLD AS o NEW AS n+FOR EACH ROW+WHEN (n.col1 > 0)+BEGIN+ :n.col2 := :o.col2 + 1;+END;`++ diagnostics, err := Diagnose(context.Background(), base.DiagnoseContext{}, statement)+ require.NoError(t, err)+ require.Empty(t, diagnostics)+}diff --git a/backend/plugin/parser/plsql/omni.go b/backend/plugin/parser/plsql/omni.goindex 879d464791..b92a34e91a 100644--- a/backend/plugin/parser/plsql/omni.go+++ b/backend/plugin/parser/plsql/omni.go@@ -1,6 +1,8 @@package plsqlimport (+ "errors"+ "fmt""reflect""unicode/utf8"@@ -86,6 +88,9 @@ func ParsePLSQLOmni(sql string) (*ast.List, error) {}parsed, err := oracleparser.Parse(statement.Text)if err != nil {+ if statement.Range != nil {+ return nil, offsetOracleParseError(err, int(statement.Range.Start))+ }return nil, err}if parsed == nil {@@ -99,6 +104,16 @@ func ParsePLSQLOmni(sql string) (*ast.List, error) {return list, nil}+func offsetOracleParseError(err error, offset int) error {+ var parseErr *oracleparser.ParseError+ if !errors.As(err, &parseErr) {+ return err+ }+ adjusted := *parseErr+ adjusted.Position += offset+ return &adjusted+}+type omniLocOffsetter intfunc offsetOmniLocs(node ast.Node, offset int) {@@ -179,3 +194,25 @@ func ByteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {Column: runeCol + 1, // convert to 1-based}}++func convertOmniError(err error, stmt base.Statement) error {+ var parseErr *oracleparser.ParseError+ if !errors.As(err, &parseErr) {+ return err+ }++ pos := ByteOffsetToRunePosition(stmt.Text, parseErr.Position)+ if stmt.Start != nil {+ if pos.Line == 1 {+ pos.Column += stmt.Start.Column - 1+ }+ pos.Line += stmt.Start.Line - 1+ }++ msg := fmt.Sprintf("Syntax error at line %d:%d: %s", pos.Line, pos.Column, parseErr.Message)+ return &base.SyntaxError{+ Position: pos,+ Message: msg,+ RawMessage: parseErr.Message,+ }+}diff --git a/backend/plugin/parser/plsql/query.go b/backend/plugin/parser/plsql/query.goindex 2273bdd15f..56cdf8eb23 100644--- a/backend/plugin/parser/plsql/query.go+++ b/backend/plugin/parser/plsql/query.go@@ -1,8 +1,8 @@package plsqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/plsql"+ oracleast "github.com/bytebase/omni/oracle/ast"+ oracleparser "github.com/bytebase/omni/oracle/parser"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -14,51 +14,31 @@ func init() {// validateQuery validates the SQL statement for SQL editor.func validateQuery(statement string) (bool, bool, error) {- results, err := ParsePLSQL(statement)+ for _, segment := range oracleparser.Split(statement) {+ if segment.Kind == oracleparser.SegmentSQLPlusCommand && !segment.Empty() {+ return false, false, nil+ }+ }++ list, err := ParsePLSQLOmni(statement)if err != nil {- return false, false, err+ return false, false, convertOmniError(err, base.Statement{Text: statement})}- if len(results) == 0 {+ if list == nil || len(list.Items) == 0 {return false, false, nil}- // Validate all statements, not just the first one- l := &queryValidateListener{- validate: true,- }- for _, result := range results {- antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)- if !l.validate {+ for _, item := range list.Items {+ raw, ok := item.(*oracleast.RawStmt)+ if !ok || raw.Stmt == nil {+ return false, false, nil+ }+ switch raw.Stmt.(type) {+ case *oracleast.SelectStmt, *oracleast.ExplainPlanStmt:+ default:return false, false, nil}}return true, true, nil}--type queryValidateListener struct {- *parser.BasePlSqlParserListener-- validate bool-}--// EnterSql_script is called when production sql_script is entered.-func (l *queryValidateListener) EnterSql_script(ctx *parser.Sql_scriptContext) {- if len(ctx.AllSql_plus_command()) > 0 {- l.validate = false- }-}--// EnterUnit_statement is called when production unit_statement is entered.-func (l *queryValidateListener) EnterUnit_statement(ctx *parser.Unit_statementContext) {- if ctx.Data_manipulation_language_statements() == nil {- l.validate = false- }-}--// EnterData_manipulation_language_statements is called when production data_manipulation_language_statements is entered.-func (l *queryValidateListener) EnterData_manipulation_language_statements(ctx *parser.Data_manipulation_language_statementsContext) {- if ctx.Select_statement() == nil && ctx.Explain_statement() == nil {- l.validate = false- }-}diff --git a/backend/plugin/parser/plsql/query_test.go b/backend/plugin/parser/plsql/query_test.gonew file mode 100644index 0000000000..a6c620a5e2--- /dev/null+++ b/backend/plugin/parser/plsql/query_test.go@@ -0,0 +1,96 @@+package plsql++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestValidateSQLForEditor(t *testing.T) {+ tests := []struct {+ name string+ statement string+ valid bool+ gotAllQuery bool+ wantErr bool+ }{+ {+ name: "single select",+ statement: "SELECT * FROM t1;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "multiple selects",+ statement: "SELECT * FROM t1; SELECT * FROM t2;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "explain plan",+ statement: "EXPLAIN PLAN FOR SELECT * FROM t1;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "explain plan for update",+ statement: "EXPLAIN PLAN FOR UPDATE t1 SET c1 = 1;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "cte select",+ statement: "WITH x AS (SELECT * FROM t1) SELECT * FROM x;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "dml keywords in string and comment",+ statement: "WITH x AS (SELECT 'update delete insert' AS c1 FROM t1 /* UPDATE */) SELECT c1 FROM x;",+ valid: true,+ gotAllQuery: true,+ },+ {+ name: "mixed select and update rejected",+ statement: "SELECT * FROM t1; UPDATE t1 SET c1 = 1;",+ },+ {+ name: "cte update rejected",+ statement: "WITH x AS (SELECT * FROM t1) UPDATE t1 SET c1 = 1;",+ wantErr: true,+ },+ {+ name: "sqlplus command rejected",+ statement: "SET DEFINE OFF\nSELECT * FROM t1;",+ },+ {+ name: "update rejected",+ statement: "UPDATE t1 SET c1 = 1;",+ },+ {+ name: "create rejected",+ statement: "CREATE TABLE t1 (c1 INT);",+ },+ {+ name: "syntax error",+ statement: "SELECT 1 FROM DUAL;\nSELECT * FROM",+ wantErr: true,+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ gotValid, gotAllQuery, err := validateQuery(tc.statement)+ if tc.wantErr {+ require.Error(t, err)+ require.ErrorAs(t, err, new(*base.SyntaxError))+ return+ }+ require.NoError(t, err)+ require.Equal(t, tc.valid, gotValid)+ require.Equal(t, tc.gotAllQuery, gotAllQuery)+ })+ }+}diff --git a/backend/plugin/parser/plsql/resource_change.go b/backend/plugin/parser/plsql/resource_change.goindex fefc5f367b..3b1b842abd 100644--- a/backend/plugin/parser/plsql/resource_change.go+++ b/backend/plugin/parser/plsql/resource_change.go@@ -1,9 +1,9 @@package plsqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/plsql"- "github.com/pkg/errors"+ "strings"++ oracleast "github.com/bytebase/omni/oracle/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -18,7 +18,7 @@ func init() {func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {// currentDatabase is the same as currentSchema for Oracle.changedResources := model.NewChangedResources(dbMetadata)- l := &plsqlChangedResourceExtractListener{+ extractor := &omniChangedResourceExtractor{currentSchema: currentDatabase,dbMetadata: dbMetadata,changedResources: changedResources,@@ -26,24 +26,22 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model}for _, ast := range asts {- antlrAST, ok := base.GetANTLRAST(ast)+ omniAST, ok := ast.(*OmniAST)if !ok {- return nil, errors.New("expected ANTLR AST for Oracle")+ continue}- antlr.ParseTreeWalkerDefault.Walk(l, antlrAST.Tree)+ extractor.extract(omniAST)}return &base.ChangeSummary{ChangedResources: changedResources,- SampleDMLS: l.sampleDMLs,- DMLCount: l.dmlCount,- InsertCount: l.insertCount,+ SampleDMLS: extractor.sampleDMLs,+ DMLCount: extractor.dmlCount,+ InsertCount: extractor.insertCount,}, nil}-type plsqlChangedResourceExtractListener struct {- *parser.BasePlSqlParserListener-+type omniChangedResourceExtractor struct {currentSchema stringdbMetadata *model.DatabaseMetadatachangedResources *model.ChangedResources@@ -51,264 +49,159 @@ type plsqlChangedResourceExtractListener struct {sampleDMLs []stringdmlCount intinsertCount int-- // Internal data structure used temporarily.- text string-}--func (l *plsqlChangedResourceExtractListener) EnterUnit_statement(ctx *parser.Unit_statementContext) {- l.text = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)}-// EnterCreate_table is called when production create_table is entered.-func (l *plsqlChangedResourceExtractListener) EnterCreate_table(ctx *parser.Create_tableContext) {- var schema string- if ctx.Schema_name() != nil {- schema = NormalizeIdentifierContext(ctx.Schema_name().Identifier())- }- if schema == "" {- schema = l.currentSchema- }-- tableName := NormalizeIdentifierContext(ctx.Table_name().Identifier())- l.changedResources.AddTable(- schema,- "",- &storepb.ChangedResourceTable{- Name: tableName,- },- false)-}--// EnterDrop_table is called when production drop_table is entered.-func (l *plsqlChangedResourceExtractListener) EnterDrop_table(ctx *parser.Drop_tableContext) {- if ctx.Tableview_name() == nil {+func (e *omniChangedResourceExtractor) extract(a *OmniAST) {+ if a == nil || a.Node == nil {return}- var schema, table string- if ctx.Tableview_name().Id_expression() == nil {- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- } else {- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())- }- if schema == "" {- schema = l.currentSchema+ switch n := a.Node.(type) {+ case *oracleast.CreateTableStmt:+ e.addTable(n.Name, false)+ case *oracleast.DropStmt:+ if n.ObjectType == oracleast.OBJECT_TABLE {+ for _, name := range omniObjectNameList(n.Names) {+ e.addTable(name, true)+ }+ }+ if n.ObjectType == oracleast.OBJECT_INDEX {+ for _, name := range omniObjectNameList(n.Names) {+ e.addIndexTable(name)+ }+ }+ case *oracleast.AlterTableStmt:+ affected := true+ for _, cmd := range omniAlterTableCmdList(n.Actions) {+ if cmd.Action == oracleast.AT_RENAME {+ affected = false+ break+ }+ }+ e.addTable(n.Name, affected)+ case *oracleast.CreateIndexStmt:+ e.addTable(n.Table, false)+ case *oracleast.InsertStmt:+ e.extractInsert(n, a.Text)+ case *oracleast.UpdateStmt:+ e.addTable(n.Table, false)+ e.trackDML(a.Text)+ case *oracleast.DeleteStmt:+ e.addTable(n.Table, false)+ e.trackDML(a.Text)+ default:}-- l.changedResources.AddTable(- schema,- "",- &storepb.ChangedResourceTable{- Name: table,- },- true)}-// EnterAlter_table is called when production alter_table is entered.-func (l *plsqlChangedResourceExtractListener) EnterAlter_table(ctx *parser.Alter_tableContext) {- if ctx.Tableview_name() == nil {- return- }-- var schema, table string- if ctx.Tableview_name().Id_expression() == nil {- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- } else {- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())+func (e *omniChangedResourceExtractor) extractInsert(stmt *oracleast.InsertStmt, text string) {+ if stmt.Table != nil {+ e.addTable(stmt.Table, false)}- if schema == "" {- schema = l.currentSchema+ for _, item := range omniInsertIntoList(stmt.MultiTable) {+ e.addTable(item.Table, false)}- l.changedResources.AddTable(- schema,- "",- &storepb.ChangedResourceTable{- Name: table,- },- true)-}--// EnterAlter_table_properties is called when production alter_table_properties is entered.-func (l *plsqlChangedResourceExtractListener) EnterAlter_table_properties(ctx *parser.Alter_table_propertiesContext) {- if ctx.RENAME() == nil {+ if stmt.Table != nil && stmt.Values != nil {+ e.insertCount++return}-- // Rename table.- var schema, table string- if ctx.Tableview_name().Id_expression() == nil {- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- } else {- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())- }- if schema == "" {- schema = l.currentSchema- }-- l.changedResources.AddTable(- schema,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false)+ e.trackDML(text)}-// EnterAlter_table is called when production create_index is entered.-func (l *plsqlChangedResourceExtractListener) EnterCreate_index(ctx *parser.Create_indexContext) {- tableIndexClause := ctx.Table_index_clause()- if tableIndexClause == nil {+func (e *omniChangedResourceExtractor) addTable(name *oracleast.ObjectName, affected bool) {+ if name == nil || name.Name == "" {return}-- var schema, table string- if tableIndexClause.Tableview_name().Id_expression() == nil {- table = NormalizeIdentifierContext(tableIndexClause.Tableview_name().Identifier())- } else {- schema = NormalizeIdentifierContext(tableIndexClause.Tableview_name().Identifier())- table = NormalizeIDExpression(tableIndexClause.Tableview_name().Id_expression())- }+ schema := name.Schemaif schema == "" {- schema = l.currentSchema+ schema = e.currentSchema}-- l.changedResources.AddTable(+ e.changedResources.AddTable(schema,"",&storepb.ChangedResourceTable{- Name: table,+ Name: name.Name,},- false)+ affected,+ )}-// EnterDrop_index is called when production drop_index is entered.-func (l *plsqlChangedResourceExtractListener) EnterDrop_index(ctx *parser.Drop_indexContext) {- schema, index := NormalizeIndexName(ctx.Index_name())+func (e *omniChangedResourceExtractor) addIndexTable(name *oracleast.ObjectName) {+ if e.dbMetadata == nil || name == nil || name.Name == "" {+ return+ }+ schema := name.Schemaif schema == "" {- schema = l.currentSchema+ schema = e.currentSchema}- foundSchema := l.dbMetadata.GetSchemaMetadata(schema)+ foundSchema := e.dbMetadata.GetSchemaMetadata(schema)if foundSchema == nil {return}- foundIndex := foundSchema.GetIndex(index)+ foundIndex := foundSchema.GetIndex(name.Name)if foundIndex == nil {return}- foundTable := foundIndex.GetTableProto().GetName()-- l.changedResources.AddTable(+ e.changedResources.AddTable(schema,"",&storepb.ChangedResourceTable{- Name: foundTable,+ Name: foundIndex.GetTableProto().GetName(),},- false)+ false,+ )}-func (l *plsqlChangedResourceExtractListener) EnterInsert_statement(ctx *parser.Insert_statementContext) {- var resources []base.SchemaResource- if ctx.Single_table_insert() != nil {- resources = append(resources, l.extractTableReference(ctx.Single_table_insert().Insert_into_clause().General_table_ref())...)- }- if ctx.Multi_table_insert() != nil {- for _, item := range ctx.Multi_table_insert().AllMulti_table_element() {- resources = append(resources, l.extractTableReference(item.Insert_into_clause().General_table_ref())...)- }- if ctx.Multi_table_insert().Conditional_insert_clause() != nil {- conditionCtx := ctx.Multi_table_insert().Conditional_insert_clause()- for _, item := range conditionCtx.AllConditional_insert_when_part() {- for _, multiItem := range item.AllMulti_table_element() {- resources = append(resources, l.extractTableReference(multiItem.Insert_into_clause().General_table_ref())...)- }- }- if conditionCtx.Conditional_insert_else_part() != nil {- for _, item := range conditionCtx.Conditional_insert_else_part().AllMulti_table_element() {- resources = append(resources, l.extractTableReference(item.Insert_into_clause().General_table_ref())...)- }- }- }- }- for _, resource := range resources {- l.changedResources.AddTable(- resource.Database,- "",- &storepb.ChangedResourceTable{- Name: resource.Table,- },- false,- )+func (e *omniChangedResourceExtractor) trackDML(text string) {+ e.dmlCount+++ if len(e.sampleDMLs) < common.MaximumLintExplainSize {+ e.sampleDMLs = append(e.sampleDMLs, omniStatementText(text))}+}- if ctx.Single_table_insert() != nil && ctx.Single_table_insert().Values_clause() != nil {- // Oracle allows only one value.- // https://docs.oracle.com/en/database/other-databases/nosql-database/22.1/sqlreferencefornosql/insert-statement.html- l.insertCount++- return- }- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)+func omniStatementText(text string) string {+ text = strings.TrimSpace(text)+ if strings.HasSuffix(text, ";") {+ return text}+ return text + ";"}-func (l *plsqlChangedResourceExtractListener) EnterUpdate_statement(ctx *parser.Update_statementContext) {- // Track DMLs.- resources := l.extractTableReference(ctx.General_table_ref())- for _, resource := range resources {- l.changedResources.AddTable(- resource.Database,- "",- &storepb.ChangedResourceTable{- Name: resource.Table,- },- false,- )+func omniObjectNameList(list *oracleast.List) []*oracleast.ObjectName {+ if list == nil {+ return nil}- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)+ names := make([]*oracleast.ObjectName, 0, len(list.Items))+ for _, item := range list.Items {+ if name, ok := item.(*oracleast.ObjectName); ok {+ names = append(names, name)+ }}+ return names}-func (l *plsqlChangedResourceExtractListener) EnterDelete_statement(ctx *parser.Delete_statementContext) {- // Track DMLs.- resources := l.extractTableReference(ctx.General_table_ref())- for _, resource := range resources {- l.changedResources.AddTable(- resource.Database,- "",- &storepb.ChangedResourceTable{- Name: resource.Table,- },- false,- )+func omniAlterTableCmdList(list *oracleast.List) []*oracleast.AlterTableCmd {+ if list == nil {+ return nil}- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)+ cmds := make([]*oracleast.AlterTableCmd, 0, len(list.Items))+ for _, item := range list.Items {+ if cmd, ok := item.(*oracleast.AlterTableCmd); ok {+ cmds = append(cmds, cmd)+ }}+ return cmds}-func (l *plsqlChangedResourceExtractListener) extractTableReference(ctx parser.IGeneral_table_refContext) []base.SchemaResource {- resources := make([]base.SchemaResource, 0)- if ctx == nil {- return resources+func omniInsertIntoList(list *oracleast.List) []*oracleast.InsertIntoClause {+ if list == nil {+ return nil}-- if ctx.Dml_table_expression_clause() != nil && ctx.Dml_table_expression_clause().Tableview_name() != nil {- _, schema, table := NormalizeTableViewName(l.currentSchema, ctx.Dml_table_expression_clause().Tableview_name())- resources = append(resources, base.SchemaResource{- Database: schema,- Table: table,- })+ clauses := make([]*oracleast.InsertIntoClause, 0, len(list.Items))+ for _, item := range list.Items {+ if clause, ok := item.(*oracleast.InsertIntoClause); ok {+ clauses = append(clauses, clause)+ }}-- return resources+ return clauses}diff --git a/backend/plugin/parser/plsql/resource_change_test.go b/backend/plugin/parser/plsql/resource_change_test.goindex 87d767136d..119d2e9929 100644--- a/backend/plugin/parser/plsql/resource_change_test.go+++ b/backend/plugin/parser/plsql/resource_change_test.go@@ -39,3 +39,132 @@ func TestExtractChangedResources(t *testing.T) {require.NoError(t, err)require.Equal(t, want, got)}++func TestExtractChangedResourcesOmniSyntax(t *testing.T) {+ statement := `CREATE TABLE IF NOT EXISTS t1 (c1 INT);+DROP TABLE IF EXISTS t2;`+ changedResources := model.NewChangedResources(nil /* dbMetadata */)+ changedResources.AddTable(+ "DB",+ "",+ &storepb.ChangedResourceTable{+ Name: "T1",+ },+ false,+ )+ changedResources.AddTable(+ "DB",+ "",+ &storepb.ChangedResourceTable{+ Name: "T2",+ },+ true,+ )+ want := &base.ChangeSummary{+ ChangedResources: changedResources,+ }++ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ require.NotEmpty(t, asts)++ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)+ require.NoError(t, err)+ require.Equal(t, want, got)+}++func TestExtractChangedResourcesDMLSamples(t *testing.T) {+ statement := `INSERT INTO t1 SELECT * FROM t2;+UPDATE t1 SET c1 = 5 WHERE c2 = 1;+DELETE FROM t2 WHERE c1 = 1;`+ changedResources := model.NewChangedResources(nil /* dbMetadata */)+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T2"}, false)+ want := &base.ChangeSummary{+ ChangedResources: changedResources,+ SampleDMLS: []string{+ "INSERT INTO t1 SELECT * FROM t2;",+ "UPDATE t1 SET c1 = 5 WHERE c2 = 1;",+ "DELETE FROM t2 WHERE c1 = 1;",+ },+ DMLCount: 3,+ }++ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ require.NotEmpty(t, asts)++ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)+ require.NoError(t, err)+ require.Equal(t, want, got)+}++func TestExtractChangedResourcesDropIndexUsesMetadata(t *testing.T) {+ statement := `DROP INDEX idx_t1_c1;`+ dbMetadata := model.NewDatabaseMetadata(&storepb.DatabaseSchemaMetadata{+ Name: "DB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "DB",+ Tables: []*storepb.TableMetadata{+ {+ Name: "T1",+ Indexes: []*storepb.IndexMetadata{+ {+ Name: "IDX_T1_C1",+ },+ },+ },+ },+ },+ },+ }, nil, nil, storepb.Engine_ORACLE, true /* isObjectCaseSensitive */)+ changedResources := model.NewChangedResources(dbMetadata)+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)+ want := &base.ChangeSummary{+ ChangedResources: changedResources,+ }++ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ require.NotEmpty(t, asts)++ got, err := extractChangedResources("DB", "", dbMetadata, asts, statement)+ require.NoError(t, err)+ require.Equal(t, want, got)+}++func TestExtractChangedResourcesSkipsANTLRFallbackAST(t *testing.T) {+ statement := `UPDATE t1 SET c1 = 5;+CREATE OR REPLACE TRIGGER trg+BEFORE INSERT OR UPDATE OF col1, col2 ON tbl+REFERENCING OLD AS o NEW AS n+FOR EACH ROW+WHEN (n.col1 > 0)+BEGIN+ :n.col2 := :o.col2 + 1;+END;`+ changedResources := model.NewChangedResources(nil /* dbMetadata */)+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)+ want := &base.ChangeSummary{+ ChangedResources: changedResources,+ SampleDMLS: []string{+ "UPDATE t1 SET c1 = 5;",+ },+ DMLCount: 1,+ }++ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)+ require.NoError(t, err)+ asts := base.ExtractASTs(stmts)+ require.Len(t, asts, 2)+ _, ok := asts[1].(*base.ANTLRAST)+ require.True(t, ok)++ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)+ require.NoError(t, err)+ require.Equal(t, want, got)+}diff --git a/backend/plugin/parser/plsql/restore.go b/backend/plugin/parser/plsql/restore.goindex 5fd9f44538..b3f84d5b76 100644--- a/backend/plugin/parser/plsql/restore.go+++ b/backend/plugin/parser/plsql/restore.go@@ -5,11 +5,9 @@ import ("fmt""strings"- "github.com/antlr4-go/antlr/v4"+ oracleast "github.com/bytebase/omni/oracle/ast""github.com/pkg/errors"- parser "github.com/bytebase/parser/plsql"-"github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -30,34 +28,77 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statementreturn "", errors.Errorf("failed to extract single SQL: %v", err)}- results, err := ParsePLSQL(originalSQL)+ node, err := findFirstOracleDML(originalSQL)if err != nil {return "", err}- if len(results) == 0 {- return "", errors.New("no parse results")- }-- // For restore SQL generation, we only examine the first statement to determine- // the table structure and operation type, even if extractSQL returns multiple statements.- // This is because:- // 1. All statements modify the same table (validated during backup creation)- // 2. All statements are the same operation type (UPDATE or DELETE)- // 3. The restore SQL is generated based on the backup table contents, not by- // reversing individual statements - we just need to know the table structure- tree := results[0].Tree- if tree == nil {- return "", errors.Errorf("no parse result")+ if node == nil {+ return "", errors.New("no DML statement found in extracted SQL")}sqlForComment, truncated := common.TruncateString(originalSQL, maxCommentLength)if truncated {sqlForComment += "..."}- return doGenerate(ctx, rCtx, sqlForComment, tree, backupItem)+ return doGenerate(ctx, rCtx, sqlForComment, node, backupItem)}-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, tree antlr.Tree, backupItem *storepb.PriorBackupDetail_Item) (string, error) {+func findFirstOracleDML(statement string) (oracleast.StmtNode, error) {+ node, err := findFirstOracleDMLOnce(statement)+ if err == nil {+ return node, nil+ }+ if !strings.Contains(statement, "\n") {+ return nil, errors.Wrap(err, "failed to parse statement")+ }++ lines := strings.Split(statement, "\n")+ for start := range lines {+ var candidate strings.Builder+ for end := start; end < len(lines); end++ {+ line := strings.TrimSpace(lines[end])+ if line == "" && candidate.Len() == 0 {+ continue+ }+ if candidate.Len() > 0 {+ candidate.WriteByte('\n')+ }+ candidate.WriteString(lines[end])+ node, candidateErr := findFirstOracleDMLOnce(candidate.String())+ if candidateErr != nil {+ continue+ }+ if node != nil {+ return node, nil+ }+ }+ }+ return nil, errors.Wrap(err, "failed to parse statement")+}++func findFirstOracleDMLOnce(statement string) (oracleast.StmtNode, error) {+ list, err := ParsePLSQLOmni(statement)+ if err != nil {+ return nil, err+ }+ if list == nil {+ return nil, nil+ }+ for _, item := range list.Items {+ raw, ok := item.(*oracleast.RawStmt)+ if !ok || raw.Stmt == nil {+ continue+ }+ switch raw.Stmt.(type) {+ case *oracleast.UpdateStmt, *oracleast.DeleteStmt:+ return raw.Stmt, nil+ default:+ }+ }+ return nil, nil+}++func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node oracleast.StmtNode, backupItem *storepb.PriorBackupDetail_Item) (string, error) {_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)if err != nil {return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)@@ -97,16 +138,13 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment strtable: tableMetadata,isFirst: true,}- antlr.ParseTreeWalkerDefault.Walk(g, tree)- if g.err != nil {- return "", g.err+ if err := g.generate(node); err != nil {+ return "", err}return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result), nil}type generator struct {- *parser.BasePlSqlParserListener-backupDatabase stringbackupTable stringoriginalDatabase string@@ -118,15 +156,26 @@ type generator struct {ctx context.ContextrCtx base.RestoreContextresult string- err error}-func (g *generator) EnterDelete_statement(ctx *parser.Delete_statementContext) {- if !IsTopLevelStatement(ctx.GetParent()) || !g.isFirst {- return+func (g *generator) generate(node oracleast.StmtNode) error {+ if !g.isFirst {+ return nil}-g.isFirst = false++ switch n := node.(type) {+ case *oracleast.DeleteStmt:+ g.generateDelete()+ case *oracleast.UpdateStmt:+ return g.generateUpdate(n)+ default:+ return errors.Errorf("unexpected statement type: %T", node)+ }+ return nil+}++func (g *generator) generateDelete() {columnList := quoteOracleColumns(g.restorableColumns())g.result = fmt.Sprintf(`INSERT INTO "%s"."%s" (%s) SELECT %s FROM "%s"."%s";`, g.originalDatabase, g.originalTable, columnList, columnList, g.backupDatabase, g.backupTable)}@@ -186,129 +235,105 @@ func (g *generator) findDisjointUniqueKey(columns []string) ([]string, error) {return nil, errors.Errorf("no disjoint unique key found for %s.%s", g.originalDatabase, g.originalTable)}-func (g *generator) EnterUpdate_statement(ctx *parser.Update_statementContext) {- if !IsTopLevelStatement(ctx.GetParent()) || !g.isFirst {- return- }-- g.isFirst = false-- l := &updateElemListener{}- antlr.ParseTreeWalkerDefault.Walk(l, ctx)- if l.err != nil {- g.err = l.err- return- }-- uk, err := g.findDisjointUniqueKey(l.result)+func (g *generator) generateUpdate(stmt *oracleast.UpdateStmt) error {+ updateColumns := extractOmniUpdateColumns(stmt)+ uk, err := g.findDisjointUniqueKey(updateColumns)if err != nil {- g.err = err- return+ return err}var buf strings.Builderif _, err := fmt.Fprintf(&buf, "MERGE INTO \"%s\".\"%s\" t\nUSING \"%s\".\"%s\" b\n ON(", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable); err != nil {- g.err = err- return+ return err}for i, column := range uk {if i > 0 {if _, err := fmt.Fprint(&buf, " AND"); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprintf(&buf, " t.\"%s\" = b.\"%s\"", column, column); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprint(&buf, ")\nWHEN MATCHED THEN\n UPDATE SET"); err != nil {- g.err = err- return+ return err}- for i, field := range l.result {+ for i, field := range updateColumns {if i > 0 {if _, err := fmt.Fprint(&buf, ","); err != nil {- g.err = err- return+ return err}}// The field is written by user and no need to escape.if _, err := fmt.Fprintf(&buf, " t.\"%s\" = b.\"%s\"", field, field); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprint(&buf, "\nWHEN NOT MATCHED THEN\n INSERT ("); err != nil {- g.err = err- return+ return err}for i, column := range g.restorableColumns() {if i > 0 {if _, err := fmt.Fprint(&buf, ", "); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprintf(&buf, "\"%s\"", column); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprint(&buf, ") VALUES ("); err != nil {- g.err = err- return+ return err}for i, column := range g.restorableColumns() {if i > 0 {if _, err := fmt.Fprint(&buf, ", "); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprintf(&buf, "b.\"%s\"", column); err != nil {- g.err = err- return+ return err}}if _, err := fmt.Fprint(&buf, ");"); err != nil {- g.err = err- return+ return err}g.result = buf.String()+ return nil}-type updateElemListener struct {- *parser.BasePlSqlParserListener-- result []string- err error+func extractOmniUpdateColumns(stmt *oracleast.UpdateStmt) []string {+ var result []string+ if stmt == nil || stmt.SetClauses == nil {+ return result+ }+ for _, item := range stmt.SetClauses.Items {+ clause, ok := item.(*oracleast.SetClause)+ if !ok {+ continue+ }+ if clause.Column != nil {+ result = append(result, clause.Column.Column)+ }+ for _, column := range omniColumnRefList(clause.Columns) {+ result = append(result, column.Column)+ }+ }+ return result}-func (l *updateElemListener) EnterColumn_based_update_set_clause(ctx *parser.Column_based_update_set_clauseContext) {- if l.err != nil {- return+func omniColumnRefList(list *oracleast.List) []*oracleast.ColumnRef {+ if list == nil {+ return nil}- if ctx.Column_name() != nil {- _, _, columnName, err := plsqlNormalizeColumnName("", ctx.Column_name())- if err != nil {- l.err = errors.Wrapf(err, "failed to normalize column name")- return- }- l.result = append(l.result, columnName)- return- }- if ctx.Paren_column_list() != nil {- for _, column := range ctx.Paren_column_list().Column_list().AllColumn_name() {- _, _, columnName, err := plsqlNormalizeColumnName("", column)- if err != nil {- l.err = errors.Wrapf(err, "failed to normalize column name")- return- }- l.result = append(l.result, columnName)+ result := make([]*oracleast.ColumnRef, 0, len(list.Items))+ for _, item := range list.Items {+ if column, ok := item.(*oracleast.ColumnRef); ok {+ result = append(result, column)}}+ return result}func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {@@ -345,6 +370,9 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (scontainsSourceTable := falsetables, err := prepareTransformation(sourceDatabase, list[i].Text)if err != nil {+ if err == errNoBackupableDML {+ continue+ }return "", errors.Wrap(err, "failed to prepare transformation")}for _, table := range tables {@@ -354,11 +382,14 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (s}}if containsSourceTable {- result = append(result, list[i].Text)+ result = append(result, normalizeExtractedRestoreSQL(list[i].Text))}}- // Statements include their leading whitespace and trailing semicolons.- return strings.Join(result, ""), nil+ return strings.Join(result, "\n"), nil+}++func normalizeExtractedRestoreSQL(statement string) string {+ return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(statement), ";"))}func equalOrLess(a, b *storepb.Position) bool {diff --git a/backend/plugin/parser/plsql/restore_test.go b/backend/plugin/parser/plsql/restore_test.goindex 94be8db12e..3b6a26cfaf 100644--- a/backend/plugin/parser/plsql/restore_test.go+++ b/backend/plugin/parser/plsql/restore_test.go@@ -78,6 +78,124 @@ func TestRestore(t *testing.T) {}}+func TestRestoreOmniBoundaryCases(t *testing.T) {+ t.Run("multi-line update", func(t *testing.T) {+ input := `UPDATE test+SET c1 = 1+WHERE c1 = 1;+UPDATE test+SET c1 = 2+WHERE c1 = 2;`++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(0, math.MaxInt32))+ require.NoError(t, err)+ require.Equal(t, `/*+Original SQL:+UPDATE test+SET c1 = 1+WHERE c1 = 1+UPDATE test+SET c1 = 2+WHERE c1 = 2+*/+MERGE INTO "DB"."TEST" t+USING "bbarchive"."prefix_1_test" b+ ON( t."A" = b."A")+WHEN MATCHED THEN+ UPDATE SET t."C1" = b."C1"+WHEN NOT MATCHED THEN+ INSERT ("A", "B", "C") VALUES (b."A", b."B", b."C");`, result)+ })++ t.Run("position range selects update", func(t *testing.T) {+ input := `DELETE FROM test WHERE c1 = 1;+UPDATE test SET c1 = 2 WHERE c1 = 2;`++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(2, 2))+ require.NoError(t, err)+ require.Contains(t, result, "UPDATE test SET c1 = 2 WHERE c1 = 2")+ require.NotContains(t, result, "DELETE FROM test WHERE c1 = 1")+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)+ })++ t.Run("same-line multi statement", func(t *testing.T) {+ input := `UPDATE test SET c1 = 1 WHERE c1 = 1; UPDATE test SET c1 = 2 WHERE c1 = 2;`++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(0, math.MaxInt32))+ require.NoError(t, err)+ require.Contains(t, result, "UPDATE test SET c1 = 1 WHERE c1 = 1\nUPDATE test SET c1 = 2 WHERE c1 = 2")+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)+ })++ t.Run("range skips non backupable statements", func(t *testing.T) {+ input := `INSERT INTO test (a, b, c) VALUES (1, 1, 1);+UPDATE test SET c1 = 2 WHERE c1 = 2;+CREATE TABLE unrelated (id NUMBER);`++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(0, math.MaxInt32))+ require.NoError(t, err)+ require.Contains(t, result, "UPDATE test SET c1 = 2 WHERE c1 = 2")+ require.NotContains(t, result, "INSERT INTO test")+ require.NotContains(t, result, "CREATE TABLE unrelated")+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)+ })++ t.Run("multi-line delete", func(t *testing.T) {+ input := `DELETE FROM test+WHERE c1 = 1;`++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(0, math.MaxInt32))+ require.NoError(t, err)+ require.Equal(t, `/*+Original SQL:+DELETE FROM test+WHERE c1 = 1+*/+INSERT INTO "DB"."TEST" ("A", "B", "C") SELECT "A", "B", "C" FROM "bbarchive"."prefix_1_test";`, result)+ })++ t.Run("no disjoint unique key", func(t *testing.T) {+ input := `UPDATE test SET a = 1 WHERE c1 = 1;`++ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,+ }, input, restoreBackupItem(0, math.MaxInt32))+ require.Error(t, err)+ require.Contains(t, err.Error(), "no disjoint unique key found")+ })+}++func restoreBackupItem(startLine, endLine int32) *store.PriorBackupDetail_Item {+ return &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/DB",+ Table: "TEST",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{+ Line: startLine,+ Column: 0,+ },+ EndPosition: &store.Position{+ Line: endLine,+ Column: math.MaxInt32,+ },+ }+}+func fixedMockDatabaseMetadataGetter(_ context.Context, _ string, database string) (string, *model.DatabaseMetadata, error) {return database, model.NewDatabaseMetadata(&store.DatabaseSchemaMetadata{Name: database,diff --git a/backend/plugin/parser/plsql/util.go b/backend/plugin/parser/plsql/util.goindex 198df54d4e..e61fc70c1e 100644--- a/backend/plugin/parser/plsql/util.go+++ b/backend/plugin/parser/plsql/util.go@@ -1,10 +1,12 @@package plsqlimport (+ "github.com/antlr4-go/antlr/v4""github.com/bytebase/parser/plsql""github.com/pkg/errors")+//nolint:unusedfunc plsqlNormalizeColumnName(currentSchema string, ctx plsql.IColumn_nameContext) (string, string, string, error) {var buf []stringbuf = append(buf, NormalizeIdentifierContext(ctx.Identifier()))@@ -64,3 +66,17 @@ func NormalizeQuotedString(ctx plsql.IQuoted_stringContext) string {raw := ctx.GetText()return raw[1 : len(raw)-1]}++func IsTopLevelStatement(ctx antlr.Tree) bool {+ if ctx == nil {+ return true+ }+ switch ctx := ctx.(type) {+ case *plsql.Unit_statementContext, *plsql.Sql_scriptContext:+ return true+ case *plsql.Data_manipulation_language_statementsContext:+ return IsTopLevelStatement(ctx.GetParent())+ default:+ return false+ }+}