Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser/plsql): migrate rollback paths to omni (#20451)

rebelice2d27eceb1a11
diff --git a/backend/plugin/parser/plsql/backup.go b/backend/plugin/parser/plsql/backup.go
index c88bd46bb3..b0a057477b 100644
--- a/backend/plugin/parser/plsql/backup.go
+++ b/backend/plugin/parser/plsql/backup.go
@@ -3,14 +3,13 @@ package plsql
import (
"context"
"fmt"
+ "reflect"
"slices"
"strings"
- "github.com/antlr4-go/antlr/v4"
+ oracleast "github.com/bytebase/omni/oracle/ast"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/plsql"
-
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -21,6 +20,8 @@ const (
maxTableNameLengthBefore12_2 = 30
)
+var errNoBackupableDML = errors.New("no parse results")
+
func init() {
base.RegisterTransformDMLToSelect(store.Engine_ORACLE, TransformDMLToSelect)
}
@@ -44,11 +45,13 @@ type TableReference struct {
}
type statementInfo struct {
- offset int
- statement string
- tree antlr.ParserRuleContext
- table *TableReference
- baseLine int
+ offset int
+ statement string
+ node oracleast.StmtNode
+ table *TableReference
+ startPosition *store.Position
+ endPosition *store.Position
+ fullSQL string
}
// TransformDMLToSelect transforms DML statement to SELECT statement.
@@ -160,7 +163,7 @@ func generateSQLForTable(ctx base.TransformContext, statementInfoList []statemen
}
}
}
- if err := writeSuffixSelectClause(&buf, info.tree); err != nil {
+ if err := writeSuffixSelectClause(&buf, info.node, info.fullSQL); err != nil {
return nil, errors.Wrap(err, "failed to write suffix select clause")
}
}
@@ -174,188 +177,240 @@ func generateSQLForTable(ctx base.TransformContext, statementInfoList []statemen
SourceSchema: table.Schema,
SourceTableName: table.Table,
TargetTableName: targetTable,
- StartPosition: &store.Position{
- Line: int32(statementInfoList[0].tree.GetStart().GetLine() + statementInfoList[0].baseLine),
- Column: int32(statementInfoList[0].tree.GetStart().GetColumn()),
- },
- EndPosition: &store.Position{
- Line: int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetLine() + statementInfoList[len(statementInfoList)-1].baseLine),
- Column: int32(statementInfoList[len(statementInfoList)-1].tree.GetStop().GetColumn()),
- },
+ StartPosition: zeroBasedColumnPosition(statementInfoList[0].startPosition),
+ EndPosition: zeroBasedColumnPosition(statementInfoList[len(statementInfoList)-1].endPosition),
}, nil
}
-func writeSuffixSelectClause(buf *strings.Builder, tree antlr.Tree) error {
- extractor := &suffixSelectClauseExtractor{
- buf: buf,
+func prepareTransformation(databaseName, statement string) ([]statementInfo, error) {
+ statements, err := SplitSQL(statement)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to split PLSQL")
}
- antlr.ParseTreeWalkerDefault.Walk(extractor, tree)
- return extractor.err
-}
-
-type suffixSelectClauseExtractor struct {
- *parser.BasePlSqlParserListener
-
- buf *strings.Builder
- err error
-}
-func (e *suffixSelectClauseExtractor) EnterDelete_statement(ctx *parser.Delete_statementContext) {
- if e.err != nil || !IsTopLevelStatement(ctx.GetParent()) {
- return
+ var result []statementInfo
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
+ for i, stmt := range statements {
+ if stmt.Empty {
+ continue
+ }
+ list, err := ParsePLSQLOmni(stmt.Text)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse PLSQL")
+ }
+ for _, item := range list.Items {
+ raw, ok := item.(*oracleast.RawStmt)
+ if !ok || raw.Stmt == nil {
+ continue
+ }
+ info := extractOmniDML(databaseName, raw.Stmt, stmt.Text)
+ if info == nil {
+ continue
+ }
+ info.offset = i
+ if stmt.Range != nil {
+ info.startPosition = positionMapper.Position(int(stmt.Range.Start) + raw.Loc.Start)
+ info.endPosition = positionMapper.Position(int(stmt.Range.Start) + max(raw.Loc.End-1, raw.Loc.Start))
+ } else {
+ info.startPosition = stmt.Start
+ info.endPosition = stmt.End
+ }
+ info.fullSQL = stmt.Text
+ result = append(result, *info)
+ }
}
- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.General_table_ref())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+ if len(result) == 0 {
+ return nil, errNoBackupableDML
}
+ return result, nil
+}
- if ctx.Where_clause() != nil {
- if _, err := e.buf.WriteString(" "); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+func extractOmniDML(databaseName string, node oracleast.StmtNode, fullSQL string) *statementInfo {
+ switch n := node.(type) {
+ case *oracleast.DeleteStmt:
+ table := omniDMLTableReference(databaseName, n.Table, n.Alias, StatementTypeDelete)
+ if table == nil {
+ return nil
+ }
+ return &statementInfo{
+ statement: extractOmniStatementText(fullSQL, n.Loc),
+ node: n,
+ table: table,
+ }
+ case *oracleast.UpdateStmt:
+ table := omniDMLTableReference(databaseName, n.Table, n.Alias, StatementTypeUpdate)
+ if table == nil {
+ return nil
}
- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+ return &statementInfo{
+ statement: extractOmniStatementText(fullSQL, n.Loc),
+ node: n,
+ table: table,
}
+ default:
+ return nil
}
}
-func (e *suffixSelectClauseExtractor) EnterUpdate_statement(ctx *parser.Update_statementContext) {
- if e.err != nil || !IsTopLevelStatement(ctx.GetParent()) {
- return
+func omniDMLTableReference(databaseName string, name *oracleast.ObjectName, alias *oracleast.Alias, statementType StatementType) *TableReference {
+ if name == nil || name.Name == "" {
+ return nil
}
-
- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.General_table_ref())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+ schema := name.Schema
+ hasSchema := schema != ""
+ if schema == "" {
+ schema = databaseName
+ }
+ table := &TableReference{
+ Database: schema,
+ HasSchema: hasSchema,
+ Schema: schema,
+ Table: name.Name,
+ StatementType: statementType,
+ }
+ if alias != nil {
+ table.Alias = alias.Name
}
+ return table
+}
- if ctx.Where_clause() != nil {
- if _, err := e.buf.WriteString(" "); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+func writeSuffixSelectClause(buf *strings.Builder, node oracleast.StmtNode, fullSQL string) error {
+ suffix := ""
+ switch n := node.(type) {
+ case *oracleast.UpdateStmt:
+ suffix = oracleDMLTargetText(fullSQL, n.Table, n.PartitionExt, n.Alias)
+ if n.WhereClause != nil && n.SetClauses != nil && n.SetClauses.Len() > 0 {
+ if setClause, ok := n.SetClauses.Items[n.SetClauses.Len()-1].(*oracleast.SetClause); ok {
+ whereStart := setClause.Loc.End
+ if fromText, fromEnd, ok := oracleListText(fullSQL, n.FromClause); ok {
+ suffix += ", " + fromText
+ whereStart = fromEnd
+ }
+ suffix = joinOracleSuffix(suffix, oracleWhereSuffix(fullSQL, whereStart, n.WhereClause))
+ }
+ }
+ case *oracleast.DeleteStmt:
+ targetEnd := n.Table.Loc.End
+ if n.PartitionExt != nil {
+ targetEnd = n.PartitionExt.Loc.End
}
- if _, err := e.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.Where_clause())); err != nil {
- e.err = errors.Wrap(err, "failed to write to buffer")
- return
+ if n.Alias != nil {
+ targetEnd = n.Alias.Loc.End
}
+ suffix = joinOracleSuffix(oracleDMLTargetText(fullSQL, n.Table, n.PartitionExt, n.Alias), oracleWhereSuffix(fullSQL, targetEnd, n.WhereClause))
+ default:
}
+ _, err := buf.WriteString(suffix)
+ return err
}
-func prepareTransformation(databaseName, statement string) ([]statementInfo, error) {
- results, err := ParsePLSQL(statement)
- if err != nil {
- return nil, errors.Wrap(err, "failed to parse PLSQL")
+func oracleDMLTargetText(sql string, name *oracleast.ObjectName, partitionExt *oracleast.PartitionExtClause, alias *oracleast.Alias) string {
+ if name == nil {
+ return ""
}
- if len(results) == 0 {
- return nil, errors.New("no parse results")
+ end := name.Loc.End
+ if partitionExt != nil {
+ end = partitionExt.Loc.End
}
-
- extractor := &dmlExtractor{
- databaseName: databaseName,
- }
-
- // Walk each ANTLRAST tree to extract DML statements
- for _, result := range results {
- extractor.baseLine = base.GetLineOffset(result.StartPosition)
- antlr.ParseTreeWalkerDefault.Walk(extractor, result.Tree)
+ if alias != nil {
+ end = alias.Loc.End
}
+ return strings.TrimSpace(oracleLocText(sql, name.Loc.Start, end))
+}
- return extractor.dmls, nil
+func oracleDMLTrailingText(sql string, start, end int) string {
+ text := strings.TrimSpace(oracleLocText(sql, start, end))
+ return strings.TrimSpace(strings.TrimSuffix(text, ";"))
}
-func IsTopLevelStatement(ctx antlr.Tree) bool {
- if ctx == nil {
- return true
+func oracleListText(sql string, list *oracleast.List) (string, int, bool) {
+ if list == nil || list.Len() == 0 {
+ return "", 0, false
}
- switch ctx := ctx.(type) {
- case *parser.Unit_statementContext, *parser.Sql_scriptContext:
- return true
- case *parser.Data_manipulation_language_statementsContext:
- return IsTopLevelStatement(ctx.GetParent())
- default:
- return false
+ start, ok := oracleNodeLoc(list.Items[0])
+ if !ok {
+ return "", 0, false
}
+ end, ok := oracleNodeLoc(list.Items[list.Len()-1])
+ if !ok {
+ return "", 0, false
+ }
+ text := strings.TrimSpace(oracleLocText(sql, start.Start, end.End))
+ if text == "" {
+ return "", 0, false
+ }
+ return text, end.End, true
}
-type dmlExtractor struct {
- *parser.BasePlSqlParserListener
-
- databaseName string
- dmls []statementInfo
- offset int
- baseLine int
-}
-
-func (e *dmlExtractor) ExitUnit_statement(_ *parser.Unit_statementContext) {
- e.offset++
+func oracleWhereSuffix(sql string, start int, where oracleast.ExprNode) string {
+ loc, ok := oracleNodeLoc(where)
+ if !ok {
+ return ""
+ }
+ return oracleDMLTrailingText(sql, start, loc.End)
}
-func (e *dmlExtractor) ExitSql_plus_command(_ *parser.Sql_plus_commandContext) {
- e.offset++
+func oracleNodeLoc(node oracleast.Node) (oracleast.Loc, bool) {
+ if node == nil {
+ return oracleast.Loc{}, false
+ }
+ value := reflect.ValueOf(node)
+ if value.Kind() != reflect.Pointer || value.IsNil() {
+ return oracleast.Loc{}, false
+ }
+ elem := value.Elem()
+ if elem.Kind() != reflect.Struct {
+ return oracleast.Loc{}, false
+ }
+ field := elem.FieldByName("Loc")
+ if !field.IsValid() || field.Type() != reflect.TypeOf(oracleast.Loc{}) {
+ return oracleast.Loc{}, false
+ }
+ loc, ok := field.Interface().(oracleast.Loc)
+ if !ok || loc.End <= loc.Start {
+ return oracleast.Loc{}, false
+ }
+ return loc, true
}
-func (e *dmlExtractor) EnterDelete_statement(ctx *parser.Delete_statementContext) {
- if IsTopLevelStatement(ctx.GetParent()) {
- extractor := &tableExtractor{
- databaseName: e.databaseName,
- }
- antlr.ParseTreeWalkerDefault.Walk(extractor, ctx)
- extractor.table.StatementType = StatementTypeDelete
-
- e.dmls = append(e.dmls, statementInfo{
- offset: e.offset,
- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- tree: ctx,
- table: extractor.table,
- baseLine: e.baseLine,
- })
+func oracleLocText(sql string, start, end int) string {
+ if start < 0 {
+ start = 0
+ }
+ if end <= 0 || end > len(sql) {
+ end = len(sql)
+ }
+ if start >= end || start >= len(sql) {
+ return ""
}
+ return sql[start:end]
}
-func (e *dmlExtractor) EnterUpdate_statement(ctx *parser.Update_statementContext) {
- if IsTopLevelStatement(ctx.GetParent()) {
- extractor := &tableExtractor{
- databaseName: e.databaseName,
+func joinOracleSuffix(parts ...string) string {
+ var nonEmpty []string
+ for _, part := range parts {
+ if part != "" {
+ nonEmpty = append(nonEmpty, part)
}
- antlr.ParseTreeWalkerDefault.Walk(extractor, ctx)
- extractor.table.StatementType = StatementTypeUpdate
-
- e.dmls = append(e.dmls, statementInfo{
- offset: e.offset,
- statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- tree: ctx,
- table: extractor.table,
- baseLine: e.baseLine,
- })
}
+ return strings.Join(nonEmpty, " ")
}
-type tableExtractor struct {
- *parser.BasePlSqlParserListener
-
- databaseName string
- table *TableReference
+func extractOmniStatementText(sql string, loc oracleast.Loc) string {
+ return strings.TrimSpace(strings.TrimSuffix(oracleLocText(sql, loc.Start, loc.End), ";"))
}
-func (e *tableExtractor) EnterGeneral_table_ref(ctx *parser.General_table_refContext) {
- dmlTableExpr := ctx.Dml_table_expression_clause()
- if dmlTableExpr != nil && dmlTableExpr.Tableview_name() != nil {
- _, schemaName, tableName := NormalizeTableViewName("", dmlTableExpr.Tableview_name())
- e.table = &TableReference{
- Database: schemaName,
- HasSchema: true,
- Schema: schemaName,
- Table: tableName,
- }
- if schemaName == "" {
- e.table.Schema = e.databaseName
- e.table.HasSchema = false
- }
- if ctx.Table_alias() != nil {
- e.table.Alias = NormalizeTableAlias(ctx.Table_alias())
- }
+func zeroBasedColumnPosition(pos *store.Position) *store.Position {
+ if pos == nil {
+ return nil
+ }
+ column := pos.Column
+ if column > 0 {
+ column--
+ }
+ return &store.Position{
+ Line: pos.Line,
+ Column: column,
}
}
diff --git a/backend/plugin/parser/plsql/backup_test.go b/backend/plugin/parser/plsql/backup_test.go
index b18be8688d..3c1876ced4 100644
--- a/backend/plugin/parser/plsql/backup_test.go
+++ b/backend/plugin/parser/plsql/backup_test.go
@@ -70,3 +70,81 @@ func TestBackup(t *testing.T) {
yamltest.Record(t, filepath, tests)
}
}
+
+func TestBackupOmniBoundaryCases(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ wantSQL string
+ }{
+ {
+ name: "update returning is not copied into select suffix",
+ input: "UPDATE test SET c1 = 1 WHERE c1 = 2 RETURNING c1 INTO :old_c1;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test WHERE c1 = 2;`,
+ },
+ {
+ name: "delete returning is not copied into select suffix",
+ input: "DELETE FROM test WHERE c1 = 2 RETURNING c1 INTO :old_c1;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test WHERE c1 = 2;`,
+ },
+ {
+ name: "update log errors is not copied into select suffix",
+ input: "UPDATE test SET c1 = 1 WHERE c1 = 2 LOG ERRORS INTO err$_test REJECT LIMIT UNLIMITED;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test WHERE c1 = 2;`,
+ },
+ {
+ name: "where keyword in block comment",
+ input: "UPDATE test SET c1 = 1 WHERE /* WHERE */ c1 = 2;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test WHERE /* WHERE */ c1 = 2;`,
+ },
+ {
+ name: "where keyword in string literal",
+ input: "DELETE FROM test WHERE note = 'WHERE should stay literal';",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test WHERE note = 'WHERE should stay literal';`,
+ },
+ {
+ name: "update without where backs up whole table",
+ input: "UPDATE test SET c1 = 1;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test;`,
+ },
+ {
+ name: "delete without where backs up whole table",
+ input: "DELETE FROM test;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test;`,
+ },
+ {
+ name: "update partition target",
+ input: "UPDATE test PARTITION (p1) SET c1 = 1 WHERE c1 = 2;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test PARTITION (p1) WHERE c1 = 2;`,
+ },
+ {
+ name: "delete subpartition target",
+ input: "DELETE FROM test SUBPARTITION (sp1) WHERE c1 = 2;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "TEST".* FROM test SUBPARTITION (sp1) WHERE c1 = 2;`,
+ },
+ {
+ name: "update from appends source table list",
+ input: "UPDATE test t SET c1 = s.c1 FROM source_table s WHERE t.id = s.id;",
+ wantSQL: `CREATE TABLE "backupDB"."rollback_TEST_DB" AS
+ SELECT "T".* FROM test t, source_table s WHERE t.id = s.id;`,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ result, err := TransformDMLToSelect(context.Background(), base.TransformContext{}, tc.input, "DB", "backupDB", "rollback")
+ require.NoError(t, err)
+ require.Len(t, result, 1)
+ require.Equal(t, tc.wantSQL, result[0].Statement)
+ })
+ }
+}
diff --git a/backend/plugin/parser/plsql/diagnose.go b/backend/plugin/parser/plsql/diagnose.go
index 5ddef63c5b..4fae75715b 100644
--- a/backend/plugin/parser/plsql/diagnose.go
+++ b/backend/plugin/parser/plsql/diagnose.go
@@ -2,12 +2,10 @@ package plsql
import (
"context"
+ "errors"
"strings"
"unicode"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/plsql"
-
"github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -26,7 +24,6 @@ func Diagnose(_ context.Context, _ base.DiagnoseContext, statement string) ([]ba
return diagnostics, nil
}
-// ParsePLSQL parses the given PLSQL.
func parsePLSQLStatement(statement string) *base.SyntaxError {
trimmedStatement := strings.TrimRightFunc(statement, unicode.IsSpace)
if len(trimmedStatement) > 0 && !strings.HasSuffix(trimmedStatement, ";") {
@@ -35,32 +32,18 @@ func parsePLSQLStatement(statement string) *base.SyntaxError {
statement += ";"
}
- lexer := parser.NewPlSqlLexer(antlr.NewInputStream(statement))
- stream := antlr.NewCommonTokenStream(lexer, 0)
- p := parser.NewPlSqlParser(stream)
- p.SetVersion12(true)
-
- lexerErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- lexer.RemoveErrorListeners()
- lexer.AddErrorListener(lexerErrorListener)
-
- parserErrorListener := &base.ParseErrorListener{
- Statement: statement,
- }
- p.RemoveErrorListeners()
- p.AddErrorListener(parserErrorListener)
-
- p.BuildParseTrees = false
- _ = p.Sql_script()
-
- if lexerErrorListener.Err != nil {
- return lexerErrorListener.Err
- }
-
- if parserErrorListener.Err != nil {
- return parserErrorListener.Err
+ if _, err := ParsePLSQLOmni(statement); err != nil {
+ if _, fallbackErr := ParsePLSQL(statement); fallbackErr == nil {
+ return nil
+ }
+ var syntaxErr *base.SyntaxError
+ if errors.As(convertOmniError(err, base.Statement{Text: statement}), &syntaxErr) {
+ return syntaxErr
+ }
+ return &base.SyntaxError{
+ Position: &store.Position{Line: 1, Column: 1},
+ Message: err.Error(),
+ }
}
return nil
}
diff --git a/backend/plugin/parser/plsql/diagnose_test.go b/backend/plugin/parser/plsql/diagnose_test.go
new file mode 100644
index 0000000000..06ef431383
--- /dev/null
+++ b/backend/plugin/parser/plsql/diagnose_test.go
@@ -0,0 +1,58 @@
+package plsql
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestDiagnoseUsesOmniParseErrorPosition(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ line uint32
+ }{
+ {
+ name: "single line",
+ statement: "SELECT * FROM",
+ line: 0,
+ },
+ {
+ name: "second sql statement",
+ statement: "SELECT 1 FROM DUAL;\n\nSELECT * FROM",
+ line: 2,
+ },
+ {
+ name: "line after non-ascii",
+ statement: "SELECT '你好' FROM DUAL;\nSELECT * FROM",
+ line: 1,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ diagnostics, err := Diagnose(context.Background(), base.DiagnoseContext{}, tc.statement)
+ require.NoError(t, err)
+ require.Len(t, diagnostics, 1)
+ require.Equal(t, tc.line, diagnostics[0].Range.Start.Line)
+ })
+ }
+}
+
+func TestDiagnoseFallsBackToANTLR(t *testing.T) {
+ statement := `CREATE OR REPLACE TRIGGER trg
+BEFORE INSERT OR UPDATE OF col1, col2 ON tbl
+REFERENCING OLD AS o NEW AS n
+FOR EACH ROW
+WHEN (n.col1 > 0)
+BEGIN
+ :n.col2 := :o.col2 + 1;
+END;`
+
+ diagnostics, err := Diagnose(context.Background(), base.DiagnoseContext{}, statement)
+ require.NoError(t, err)
+ require.Empty(t, diagnostics)
+}
diff --git a/backend/plugin/parser/plsql/omni.go b/backend/plugin/parser/plsql/omni.go
index 879d464791..b92a34e91a 100644
--- a/backend/plugin/parser/plsql/omni.go
+++ b/backend/plugin/parser/plsql/omni.go
@@ -1,6 +1,8 @@
package plsql
import (
+ "errors"
+ "fmt"
"reflect"
"unicode/utf8"
@@ -86,6 +88,9 @@ func ParsePLSQLOmni(sql string) (*ast.List, error) {
}
parsed, err := oracleparser.Parse(statement.Text)
if err != nil {
+ if statement.Range != nil {
+ return nil, offsetOracleParseError(err, int(statement.Range.Start))
+ }
return nil, err
}
if parsed == nil {
@@ -99,6 +104,16 @@ func ParsePLSQLOmni(sql string) (*ast.List, error) {
return list, nil
}
+func offsetOracleParseError(err error, offset int) error {
+ var parseErr *oracleparser.ParseError
+ if !errors.As(err, &parseErr) {
+ return err
+ }
+ adjusted := *parseErr
+ adjusted.Position += offset
+ return &adjusted
+}
+
type omniLocOffsetter int
func offsetOmniLocs(node ast.Node, offset int) {
@@ -179,3 +194,25 @@ func ByteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
Column: runeCol + 1, // convert to 1-based
}
}
+
+func convertOmniError(err error, stmt base.Statement) error {
+ var parseErr *oracleparser.ParseError
+ if !errors.As(err, &parseErr) {
+ return err
+ }
+
+ pos := ByteOffsetToRunePosition(stmt.Text, parseErr.Position)
+ if stmt.Start != nil {
+ if pos.Line == 1 {
+ pos.Column += stmt.Start.Column - 1
+ }
+ pos.Line += stmt.Start.Line - 1
+ }
+
+ msg := fmt.Sprintf("Syntax error at line %d:%d: %s", pos.Line, pos.Column, parseErr.Message)
+ return &base.SyntaxError{
+ Position: pos,
+ Message: msg,
+ RawMessage: parseErr.Message,
+ }
+}
diff --git a/backend/plugin/parser/plsql/query.go b/backend/plugin/parser/plsql/query.go
index 2273bdd15f..56cdf8eb23 100644
--- a/backend/plugin/parser/plsql/query.go
+++ b/backend/plugin/parser/plsql/query.go
@@ -1,8 +1,8 @@
package plsql
import (
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/plsql"
+ oracleast "github.com/bytebase/omni/oracle/ast"
+ oracleparser "github.com/bytebase/omni/oracle/parser"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -14,51 +14,31 @@ func init() {
// validateQuery validates the SQL statement for SQL editor.
func validateQuery(statement string) (bool, bool, error) {
- results, err := ParsePLSQL(statement)
+ for _, segment := range oracleparser.Split(statement) {
+ if segment.Kind == oracleparser.SegmentSQLPlusCommand && !segment.Empty() {
+ return false, false, nil
+ }
+ }
+
+ list, err := ParsePLSQLOmni(statement)
if err != nil {
- return false, false, err
+ return false, false, convertOmniError(err, base.Statement{Text: statement})
}
- if len(results) == 0 {
+ if list == nil || len(list.Items) == 0 {
return false, false, nil
}
- // Validate all statements, not just the first one
- l := &queryValidateListener{
- validate: true,
- }
- for _, result := range results {
- antlr.ParseTreeWalkerDefault.Walk(l, result.Tree)
- if !l.validate {
+ for _, item := range list.Items {
+ raw, ok := item.(*oracleast.RawStmt)
+ if !ok || raw.Stmt == nil {
+ return false, false, nil
+ }
+ switch raw.Stmt.(type) {
+ case *oracleast.SelectStmt, *oracleast.ExplainPlanStmt:
+ default:
return false, false, nil
}
}
return true, true, nil
}
-
-type queryValidateListener struct {
- *parser.BasePlSqlParserListener
-
- validate bool
-}
-
-// EnterSql_script is called when production sql_script is entered.
-func (l *queryValidateListener) EnterSql_script(ctx *parser.Sql_scriptContext) {
- if len(ctx.AllSql_plus_command()) > 0 {
- l.validate = false
- }
-}
-
-// EnterUnit_statement is called when production unit_statement is entered.
-func (l *queryValidateListener) EnterUnit_statement(ctx *parser.Unit_statementContext) {
- if ctx.Data_manipulation_language_statements() == nil {
- l.validate = false
- }
-}
-
-// EnterData_manipulation_language_statements is called when production data_manipulation_language_statements is entered.
-func (l *queryValidateListener) EnterData_manipulation_language_statements(ctx *parser.Data_manipulation_language_statementsContext) {
- if ctx.Select_statement() == nil && ctx.Explain_statement() == nil {
- l.validate = false
- }
-}
diff --git a/backend/plugin/parser/plsql/query_test.go b/backend/plugin/parser/plsql/query_test.go
new file mode 100644
index 0000000000..a6c620a5e2
--- /dev/null
+++ b/backend/plugin/parser/plsql/query_test.go
@@ -0,0 +1,96 @@
+package plsql
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestValidateSQLForEditor(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ valid bool
+ gotAllQuery bool
+ wantErr bool
+ }{
+ {
+ name: "single select",
+ statement: "SELECT * FROM t1;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "multiple selects",
+ statement: "SELECT * FROM t1; SELECT * FROM t2;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "explain plan",
+ statement: "EXPLAIN PLAN FOR SELECT * FROM t1;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "explain plan for update",
+ statement: "EXPLAIN PLAN FOR UPDATE t1 SET c1 = 1;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "cte select",
+ statement: "WITH x AS (SELECT * FROM t1) SELECT * FROM x;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "dml keywords in string and comment",
+ statement: "WITH x AS (SELECT 'update delete insert' AS c1 FROM t1 /* UPDATE */) SELECT c1 FROM x;",
+ valid: true,
+ gotAllQuery: true,
+ },
+ {
+ name: "mixed select and update rejected",
+ statement: "SELECT * FROM t1; UPDATE t1 SET c1 = 1;",
+ },
+ {
+ name: "cte update rejected",
+ statement: "WITH x AS (SELECT * FROM t1) UPDATE t1 SET c1 = 1;",
+ wantErr: true,
+ },
+ {
+ name: "sqlplus command rejected",
+ statement: "SET DEFINE OFF\nSELECT * FROM t1;",
+ },
+ {
+ name: "update rejected",
+ statement: "UPDATE t1 SET c1 = 1;",
+ },
+ {
+ name: "create rejected",
+ statement: "CREATE TABLE t1 (c1 INT);",
+ },
+ {
+ name: "syntax error",
+ statement: "SELECT 1 FROM DUAL;\nSELECT * FROM",
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ gotValid, gotAllQuery, err := validateQuery(tc.statement)
+ if tc.wantErr {
+ require.Error(t, err)
+ require.ErrorAs(t, err, new(*base.SyntaxError))
+ return
+ }
+ require.NoError(t, err)
+ require.Equal(t, tc.valid, gotValid)
+ require.Equal(t, tc.gotAllQuery, gotAllQuery)
+ })
+ }
+}
diff --git a/backend/plugin/parser/plsql/resource_change.go b/backend/plugin/parser/plsql/resource_change.go
index fefc5f367b..3b1b842abd 100644
--- a/backend/plugin/parser/plsql/resource_change.go
+++ b/backend/plugin/parser/plsql/resource_change.go
@@ -1,9 +1,9 @@
package plsql
import (
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/plsql"
- "github.com/pkg/errors"
+ "strings"
+
+ oracleast "github.com/bytebase/omni/oracle/ast"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -18,7 +18,7 @@ func init() {
func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {
// currentDatabase is the same as currentSchema for Oracle.
changedResources := model.NewChangedResources(dbMetadata)
- l := &plsqlChangedResourceExtractListener{
+ extractor := &omniChangedResourceExtractor{
currentSchema: currentDatabase,
dbMetadata: dbMetadata,
changedResources: changedResources,
@@ -26,24 +26,22 @@ func extractChangedResources(currentDatabase string, _ string, dbMetadata *model
}
for _, ast := range asts {
- antlrAST, ok := base.GetANTLRAST(ast)
+ omniAST, ok := ast.(*OmniAST)
if !ok {
- return nil, errors.New("expected ANTLR AST for Oracle")
+ continue
}
- antlr.ParseTreeWalkerDefault.Walk(l, antlrAST.Tree)
+ extractor.extract(omniAST)
}
return &base.ChangeSummary{
ChangedResources: changedResources,
- SampleDMLS: l.sampleDMLs,
- DMLCount: l.dmlCount,
- InsertCount: l.insertCount,
+ SampleDMLS: extractor.sampleDMLs,
+ DMLCount: extractor.dmlCount,
+ InsertCount: extractor.insertCount,
}, nil
}
-type plsqlChangedResourceExtractListener struct {
- *parser.BasePlSqlParserListener
-
+type omniChangedResourceExtractor struct {
currentSchema string
dbMetadata *model.DatabaseMetadata
changedResources *model.ChangedResources
@@ -51,264 +49,159 @@ type plsqlChangedResourceExtractListener struct {
sampleDMLs []string
dmlCount int
insertCount int
-
- // Internal data structure used temporarily.
- text string
-}
-
-func (l *plsqlChangedResourceExtractListener) EnterUnit_statement(ctx *parser.Unit_statementContext) {
- l.text = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
}
-// EnterCreate_table is called when production create_table is entered.
-func (l *plsqlChangedResourceExtractListener) EnterCreate_table(ctx *parser.Create_tableContext) {
- var schema string
- if ctx.Schema_name() != nil {
- schema = NormalizeIdentifierContext(ctx.Schema_name().Identifier())
- }
- if schema == "" {
- schema = l.currentSchema
- }
-
- tableName := NormalizeIdentifierContext(ctx.Table_name().Identifier())
- l.changedResources.AddTable(
- schema,
- "",
- &storepb.ChangedResourceTable{
- Name: tableName,
- },
- false)
-}
-
-// EnterDrop_table is called when production drop_table is entered.
-func (l *plsqlChangedResourceExtractListener) EnterDrop_table(ctx *parser.Drop_tableContext) {
- if ctx.Tableview_name() == nil {
+func (e *omniChangedResourceExtractor) extract(a *OmniAST) {
+ if a == nil || a.Node == nil {
return
}
- var schema, table string
- if ctx.Tableview_name().Id_expression() == nil {
- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- } else {
- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())
- }
- if schema == "" {
- schema = l.currentSchema
+ switch n := a.Node.(type) {
+ case *oracleast.CreateTableStmt:
+ e.addTable(n.Name, false)
+ case *oracleast.DropStmt:
+ if n.ObjectType == oracleast.OBJECT_TABLE {
+ for _, name := range omniObjectNameList(n.Names) {
+ e.addTable(name, true)
+ }
+ }
+ if n.ObjectType == oracleast.OBJECT_INDEX {
+ for _, name := range omniObjectNameList(n.Names) {
+ e.addIndexTable(name)
+ }
+ }
+ case *oracleast.AlterTableStmt:
+ affected := true
+ for _, cmd := range omniAlterTableCmdList(n.Actions) {
+ if cmd.Action == oracleast.AT_RENAME {
+ affected = false
+ break
+ }
+ }
+ e.addTable(n.Name, affected)
+ case *oracleast.CreateIndexStmt:
+ e.addTable(n.Table, false)
+ case *oracleast.InsertStmt:
+ e.extractInsert(n, a.Text)
+ case *oracleast.UpdateStmt:
+ e.addTable(n.Table, false)
+ e.trackDML(a.Text)
+ case *oracleast.DeleteStmt:
+ e.addTable(n.Table, false)
+ e.trackDML(a.Text)
+ default:
}
-
- l.changedResources.AddTable(
- schema,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- true)
}
-// EnterAlter_table is called when production alter_table is entered.
-func (l *plsqlChangedResourceExtractListener) EnterAlter_table(ctx *parser.Alter_tableContext) {
- if ctx.Tableview_name() == nil {
- return
- }
-
- var schema, table string
- if ctx.Tableview_name().Id_expression() == nil {
- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- } else {
- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())
+func (e *omniChangedResourceExtractor) extractInsert(stmt *oracleast.InsertStmt, text string) {
+ if stmt.Table != nil {
+ e.addTable(stmt.Table, false)
}
- if schema == "" {
- schema = l.currentSchema
+ for _, item := range omniInsertIntoList(stmt.MultiTable) {
+ e.addTable(item.Table, false)
}
- l.changedResources.AddTable(
- schema,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- true)
-}
-
-// EnterAlter_table_properties is called when production alter_table_properties is entered.
-func (l *plsqlChangedResourceExtractListener) EnterAlter_table_properties(ctx *parser.Alter_table_propertiesContext) {
- if ctx.RENAME() == nil {
+ if stmt.Table != nil && stmt.Values != nil {
+ e.insertCount++
return
}
-
- // Rename table.
- var schema, table string
- if ctx.Tableview_name().Id_expression() == nil {
- table = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- } else {
- schema = NormalizeIdentifierContext(ctx.Tableview_name().Identifier())
- table = NormalizeIDExpression(ctx.Tableview_name().Id_expression())
- }
- if schema == "" {
- schema = l.currentSchema
- }
-
- l.changedResources.AddTable(
- schema,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false)
+ e.trackDML(text)
}
-// EnterAlter_table is called when production create_index is entered.
-func (l *plsqlChangedResourceExtractListener) EnterCreate_index(ctx *parser.Create_indexContext) {
- tableIndexClause := ctx.Table_index_clause()
- if tableIndexClause == nil {
+func (e *omniChangedResourceExtractor) addTable(name *oracleast.ObjectName, affected bool) {
+ if name == nil || name.Name == "" {
return
}
-
- var schema, table string
- if tableIndexClause.Tableview_name().Id_expression() == nil {
- table = NormalizeIdentifierContext(tableIndexClause.Tableview_name().Identifier())
- } else {
- schema = NormalizeIdentifierContext(tableIndexClause.Tableview_name().Identifier())
- table = NormalizeIDExpression(tableIndexClause.Tableview_name().Id_expression())
- }
+ schema := name.Schema
if schema == "" {
- schema = l.currentSchema
+ schema = e.currentSchema
}
-
- l.changedResources.AddTable(
+ e.changedResources.AddTable(
schema,
"",
&storepb.ChangedResourceTable{
- Name: table,
+ Name: name.Name,
},
- false)
+ affected,
+ )
}
-// EnterDrop_index is called when production drop_index is entered.
-func (l *plsqlChangedResourceExtractListener) EnterDrop_index(ctx *parser.Drop_indexContext) {
- schema, index := NormalizeIndexName(ctx.Index_name())
+func (e *omniChangedResourceExtractor) addIndexTable(name *oracleast.ObjectName) {
+ if e.dbMetadata == nil || name == nil || name.Name == "" {
+ return
+ }
+ schema := name.Schema
if schema == "" {
- schema = l.currentSchema
+ schema = e.currentSchema
}
- foundSchema := l.dbMetadata.GetSchemaMetadata(schema)
+ foundSchema := e.dbMetadata.GetSchemaMetadata(schema)
if foundSchema == nil {
return
}
- foundIndex := foundSchema.GetIndex(index)
+ foundIndex := foundSchema.GetIndex(name.Name)
if foundIndex == nil {
return
}
- foundTable := foundIndex.GetTableProto().GetName()
-
- l.changedResources.AddTable(
+ e.changedResources.AddTable(
schema,
"",
&storepb.ChangedResourceTable{
- Name: foundTable,
+ Name: foundIndex.GetTableProto().GetName(),
},
- false)
+ false,
+ )
}
-func (l *plsqlChangedResourceExtractListener) EnterInsert_statement(ctx *parser.Insert_statementContext) {
- var resources []base.SchemaResource
- if ctx.Single_table_insert() != nil {
- resources = append(resources, l.extractTableReference(ctx.Single_table_insert().Insert_into_clause().General_table_ref())...)
- }
- if ctx.Multi_table_insert() != nil {
- for _, item := range ctx.Multi_table_insert().AllMulti_table_element() {
- resources = append(resources, l.extractTableReference(item.Insert_into_clause().General_table_ref())...)
- }
- if ctx.Multi_table_insert().Conditional_insert_clause() != nil {
- conditionCtx := ctx.Multi_table_insert().Conditional_insert_clause()
- for _, item := range conditionCtx.AllConditional_insert_when_part() {
- for _, multiItem := range item.AllMulti_table_element() {
- resources = append(resources, l.extractTableReference(multiItem.Insert_into_clause().General_table_ref())...)
- }
- }
- if conditionCtx.Conditional_insert_else_part() != nil {
- for _, item := range conditionCtx.Conditional_insert_else_part().AllMulti_table_element() {
- resources = append(resources, l.extractTableReference(item.Insert_into_clause().General_table_ref())...)
- }
- }
- }
- }
- for _, resource := range resources {
- l.changedResources.AddTable(
- resource.Database,
- "",
- &storepb.ChangedResourceTable{
- Name: resource.Table,
- },
- false,
- )
+func (e *omniChangedResourceExtractor) trackDML(text string) {
+ e.dmlCount++
+ if len(e.sampleDMLs) < common.MaximumLintExplainSize {
+ e.sampleDMLs = append(e.sampleDMLs, omniStatementText(text))
}
+}
- if ctx.Single_table_insert() != nil && ctx.Single_table_insert().Values_clause() != nil {
- // Oracle allows only one value.
- // https://docs.oracle.com/en/database/other-databases/nosql-database/22.1/sqlreferencefornosql/insert-statement.html
- l.insertCount++
- return
- }
- // Track DMLs.
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
+func omniStatementText(text string) string {
+ text = strings.TrimSpace(text)
+ if strings.HasSuffix(text, ";") {
+ return text
}
+ return text + ";"
}
-func (l *plsqlChangedResourceExtractListener) EnterUpdate_statement(ctx *parser.Update_statementContext) {
- // Track DMLs.
- resources := l.extractTableReference(ctx.General_table_ref())
- for _, resource := range resources {
- l.changedResources.AddTable(
- resource.Database,
- "",
- &storepb.ChangedResourceTable{
- Name: resource.Table,
- },
- false,
- )
+func omniObjectNameList(list *oracleast.List) []*oracleast.ObjectName {
+ if list == nil {
+ return nil
}
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
+ names := make([]*oracleast.ObjectName, 0, len(list.Items))
+ for _, item := range list.Items {
+ if name, ok := item.(*oracleast.ObjectName); ok {
+ names = append(names, name)
+ }
}
+ return names
}
-func (l *plsqlChangedResourceExtractListener) EnterDelete_statement(ctx *parser.Delete_statementContext) {
- // Track DMLs.
- resources := l.extractTableReference(ctx.General_table_ref())
- for _, resource := range resources {
- l.changedResources.AddTable(
- resource.Database,
- "",
- &storepb.ChangedResourceTable{
- Name: resource.Table,
- },
- false,
- )
+func omniAlterTableCmdList(list *oracleast.List) []*oracleast.AlterTableCmd {
+ if list == nil {
+ return nil
}
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
+ cmds := make([]*oracleast.AlterTableCmd, 0, len(list.Items))
+ for _, item := range list.Items {
+ if cmd, ok := item.(*oracleast.AlterTableCmd); ok {
+ cmds = append(cmds, cmd)
+ }
}
+ return cmds
}
-func (l *plsqlChangedResourceExtractListener) extractTableReference(ctx parser.IGeneral_table_refContext) []base.SchemaResource {
- resources := make([]base.SchemaResource, 0)
- if ctx == nil {
- return resources
+func omniInsertIntoList(list *oracleast.List) []*oracleast.InsertIntoClause {
+ if list == nil {
+ return nil
}
-
- if ctx.Dml_table_expression_clause() != nil && ctx.Dml_table_expression_clause().Tableview_name() != nil {
- _, schema, table := NormalizeTableViewName(l.currentSchema, ctx.Dml_table_expression_clause().Tableview_name())
- resources = append(resources, base.SchemaResource{
- Database: schema,
- Table: table,
- })
+ clauses := make([]*oracleast.InsertIntoClause, 0, len(list.Items))
+ for _, item := range list.Items {
+ if clause, ok := item.(*oracleast.InsertIntoClause); ok {
+ clauses = append(clauses, clause)
+ }
}
-
- return resources
+ return clauses
}
diff --git a/backend/plugin/parser/plsql/resource_change_test.go b/backend/plugin/parser/plsql/resource_change_test.go
index 87d767136d..119d2e9929 100644
--- a/backend/plugin/parser/plsql/resource_change_test.go
+++ b/backend/plugin/parser/plsql/resource_change_test.go
@@ -39,3 +39,132 @@ func TestExtractChangedResources(t *testing.T) {
require.NoError(t, err)
require.Equal(t, want, got)
}
+
+func TestExtractChangedResourcesOmniSyntax(t *testing.T) {
+ statement := `CREATE TABLE IF NOT EXISTS t1 (c1 INT);
+DROP TABLE IF EXISTS t2;`
+ changedResources := model.NewChangedResources(nil /* dbMetadata */)
+ changedResources.AddTable(
+ "DB",
+ "",
+ &storepb.ChangedResourceTable{
+ Name: "T1",
+ },
+ false,
+ )
+ changedResources.AddTable(
+ "DB",
+ "",
+ &storepb.ChangedResourceTable{
+ Name: "T2",
+ },
+ true,
+ )
+ want := &base.ChangeSummary{
+ ChangedResources: changedResources,
+ }
+
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)
+ require.NoError(t, err)
+ asts := base.ExtractASTs(stmts)
+ require.NotEmpty(t, asts)
+
+ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+}
+
+func TestExtractChangedResourcesDMLSamples(t *testing.T) {
+ statement := `INSERT INTO t1 SELECT * FROM t2;
+UPDATE t1 SET c1 = 5 WHERE c2 = 1;
+DELETE FROM t2 WHERE c1 = 1;`
+ changedResources := model.NewChangedResources(nil /* dbMetadata */)
+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)
+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T2"}, false)
+ want := &base.ChangeSummary{
+ ChangedResources: changedResources,
+ SampleDMLS: []string{
+ "INSERT INTO t1 SELECT * FROM t2;",
+ "UPDATE t1 SET c1 = 5 WHERE c2 = 1;",
+ "DELETE FROM t2 WHERE c1 = 1;",
+ },
+ DMLCount: 3,
+ }
+
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)
+ require.NoError(t, err)
+ asts := base.ExtractASTs(stmts)
+ require.NotEmpty(t, asts)
+
+ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+}
+
+func TestExtractChangedResourcesDropIndexUsesMetadata(t *testing.T) {
+ statement := `DROP INDEX idx_t1_c1;`
+ dbMetadata := model.NewDatabaseMetadata(&storepb.DatabaseSchemaMetadata{
+ Name: "DB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "DB",
+ Tables: []*storepb.TableMetadata{
+ {
+ Name: "T1",
+ Indexes: []*storepb.IndexMetadata{
+ {
+ Name: "IDX_T1_C1",
+ },
+ },
+ },
+ },
+ },
+ },
+ }, nil, nil, storepb.Engine_ORACLE, true /* isObjectCaseSensitive */)
+ changedResources := model.NewChangedResources(dbMetadata)
+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)
+ want := &base.ChangeSummary{
+ ChangedResources: changedResources,
+ }
+
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)
+ require.NoError(t, err)
+ asts := base.ExtractASTs(stmts)
+ require.NotEmpty(t, asts)
+
+ got, err := extractChangedResources("DB", "", dbMetadata, asts, statement)
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+}
+
+func TestExtractChangedResourcesSkipsANTLRFallbackAST(t *testing.T) {
+ statement := `UPDATE t1 SET c1 = 5;
+CREATE OR REPLACE TRIGGER trg
+BEFORE INSERT OR UPDATE OF col1, col2 ON tbl
+REFERENCING OLD AS o NEW AS n
+FOR EACH ROW
+WHEN (n.col1 > 0)
+BEGIN
+ :n.col2 := :o.col2 + 1;
+END;`
+ changedResources := model.NewChangedResources(nil /* dbMetadata */)
+ changedResources.AddTable("DB", "", &storepb.ChangedResourceTable{Name: "T1"}, false)
+ want := &base.ChangeSummary{
+ ChangedResources: changedResources,
+ SampleDMLS: []string{
+ "UPDATE t1 SET c1 = 5;",
+ },
+ DMLCount: 1,
+ }
+
+ stmts, err := base.ParseStatements(storepb.Engine_ORACLE, statement)
+ require.NoError(t, err)
+ asts := base.ExtractASTs(stmts)
+ require.Len(t, asts, 2)
+ _, ok := asts[1].(*base.ANTLRAST)
+ require.True(t, ok)
+
+ got, err := extractChangedResources("DB", "", nil /* dbMetadata */, asts, statement)
+ require.NoError(t, err)
+ require.Equal(t, want, got)
+}
diff --git a/backend/plugin/parser/plsql/restore.go b/backend/plugin/parser/plsql/restore.go
index 5fd9f44538..b3f84d5b76 100644
--- a/backend/plugin/parser/plsql/restore.go
+++ b/backend/plugin/parser/plsql/restore.go
@@ -5,11 +5,9 @@ import (
"fmt"
"strings"
- "github.com/antlr4-go/antlr/v4"
+ oracleast "github.com/bytebase/omni/oracle/ast"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/plsql"
-
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -30,34 +28,77 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statement
return "", errors.Errorf("failed to extract single SQL: %v", err)
}
- results, err := ParsePLSQL(originalSQL)
+ node, err := findFirstOracleDML(originalSQL)
if err != nil {
return "", err
}
- if len(results) == 0 {
- return "", errors.New("no parse results")
- }
-
- // For restore SQL generation, we only examine the first statement to determine
- // the table structure and operation type, even if extractSQL returns multiple statements.
- // This is because:
- // 1. All statements modify the same table (validated during backup creation)
- // 2. All statements are the same operation type (UPDATE or DELETE)
- // 3. The restore SQL is generated based on the backup table contents, not by
- // reversing individual statements - we just need to know the table structure
- tree := results[0].Tree
- if tree == nil {
- return "", errors.Errorf("no parse result")
+ if node == nil {
+ return "", errors.New("no DML statement found in extracted SQL")
}
sqlForComment, truncated := common.TruncateString(originalSQL, maxCommentLength)
if truncated {
sqlForComment += "..."
}
- return doGenerate(ctx, rCtx, sqlForComment, tree, backupItem)
+ return doGenerate(ctx, rCtx, sqlForComment, node, backupItem)
}
-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, tree antlr.Tree, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
+func findFirstOracleDML(statement string) (oracleast.StmtNode, error) {
+ node, err := findFirstOracleDMLOnce(statement)
+ if err == nil {
+ return node, nil
+ }
+ if !strings.Contains(statement, "\n") {
+ return nil, errors.Wrap(err, "failed to parse statement")
+ }
+
+ lines := strings.Split(statement, "\n")
+ for start := range lines {
+ var candidate strings.Builder
+ for end := start; end < len(lines); end++ {
+ line := strings.TrimSpace(lines[end])
+ if line == "" && candidate.Len() == 0 {
+ continue
+ }
+ if candidate.Len() > 0 {
+ candidate.WriteByte('\n')
+ }
+ candidate.WriteString(lines[end])
+ node, candidateErr := findFirstOracleDMLOnce(candidate.String())
+ if candidateErr != nil {
+ continue
+ }
+ if node != nil {
+ return node, nil
+ }
+ }
+ }
+ return nil, errors.Wrap(err, "failed to parse statement")
+}
+
+func findFirstOracleDMLOnce(statement string) (oracleast.StmtNode, error) {
+ list, err := ParsePLSQLOmni(statement)
+ if err != nil {
+ return nil, err
+ }
+ if list == nil {
+ return nil, nil
+ }
+ for _, item := range list.Items {
+ raw, ok := item.(*oracleast.RawStmt)
+ if !ok || raw.Stmt == nil {
+ continue
+ }
+ switch raw.Stmt.(type) {
+ case *oracleast.UpdateStmt, *oracleast.DeleteStmt:
+ return raw.Stmt, nil
+ default:
+ }
+ }
+ return nil, nil
+}
+
+func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node oracleast.StmtNode, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)
if err != nil {
return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)
@@ -97,16 +138,13 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment str
table: tableMetadata,
isFirst: true,
}
- antlr.ParseTreeWalkerDefault.Walk(g, tree)
- if g.err != nil {
- return "", g.err
+ if err := g.generate(node); err != nil {
+ return "", err
}
return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result), nil
}
type generator struct {
- *parser.BasePlSqlParserListener
-
backupDatabase string
backupTable string
originalDatabase string
@@ -118,15 +156,26 @@ type generator struct {
ctx context.Context
rCtx base.RestoreContext
result string
- err error
}
-func (g *generator) EnterDelete_statement(ctx *parser.Delete_statementContext) {
- if !IsTopLevelStatement(ctx.GetParent()) || !g.isFirst {
- return
+func (g *generator) generate(node oracleast.StmtNode) error {
+ if !g.isFirst {
+ return nil
}
-
g.isFirst = false
+
+ switch n := node.(type) {
+ case *oracleast.DeleteStmt:
+ g.generateDelete()
+ case *oracleast.UpdateStmt:
+ return g.generateUpdate(n)
+ default:
+ return errors.Errorf("unexpected statement type: %T", node)
+ }
+ return nil
+}
+
+func (g *generator) generateDelete() {
columnList := quoteOracleColumns(g.restorableColumns())
g.result = fmt.Sprintf(`INSERT INTO "%s"."%s" (%s) SELECT %s FROM "%s"."%s";`, g.originalDatabase, g.originalTable, columnList, columnList, g.backupDatabase, g.backupTable)
}
@@ -186,129 +235,105 @@ func (g *generator) findDisjointUniqueKey(columns []string) ([]string, error) {
return nil, errors.Errorf("no disjoint unique key found for %s.%s", g.originalDatabase, g.originalTable)
}
-func (g *generator) EnterUpdate_statement(ctx *parser.Update_statementContext) {
- if !IsTopLevelStatement(ctx.GetParent()) || !g.isFirst {
- return
- }
-
- g.isFirst = false
-
- l := &updateElemListener{}
- antlr.ParseTreeWalkerDefault.Walk(l, ctx)
- if l.err != nil {
- g.err = l.err
- return
- }
-
- uk, err := g.findDisjointUniqueKey(l.result)
+func (g *generator) generateUpdate(stmt *oracleast.UpdateStmt) error {
+ updateColumns := extractOmniUpdateColumns(stmt)
+ uk, err := g.findDisjointUniqueKey(updateColumns)
if err != nil {
- g.err = err
- return
+ return err
}
var buf strings.Builder
if _, err := fmt.Fprintf(&buf, "MERGE INTO \"%s\".\"%s\" t\nUSING \"%s\".\"%s\" b\n ON(", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable); err != nil {
- g.err = err
- return
+ return err
}
for i, column := range uk {
if i > 0 {
if _, err := fmt.Fprint(&buf, " AND"); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprintf(&buf, " t.\"%s\" = b.\"%s\"", column, column); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprint(&buf, ")\nWHEN MATCHED THEN\n UPDATE SET"); err != nil {
- g.err = err
- return
+ return err
}
- for i, field := range l.result {
+ for i, field := range updateColumns {
if i > 0 {
if _, err := fmt.Fprint(&buf, ","); err != nil {
- g.err = err
- return
+ return err
}
}
// The field is written by user and no need to escape.
if _, err := fmt.Fprintf(&buf, " t.\"%s\" = b.\"%s\"", field, field); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprint(&buf, "\nWHEN NOT MATCHED THEN\n INSERT ("); err != nil {
- g.err = err
- return
+ return err
}
for i, column := range g.restorableColumns() {
if i > 0 {
if _, err := fmt.Fprint(&buf, ", "); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprintf(&buf, "\"%s\"", column); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprint(&buf, ") VALUES ("); err != nil {
- g.err = err
- return
+ return err
}
for i, column := range g.restorableColumns() {
if i > 0 {
if _, err := fmt.Fprint(&buf, ", "); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprintf(&buf, "b.\"%s\"", column); err != nil {
- g.err = err
- return
+ return err
}
}
if _, err := fmt.Fprint(&buf, ");"); err != nil {
- g.err = err
- return
+ return err
}
g.result = buf.String()
+ return nil
}
-type updateElemListener struct {
- *parser.BasePlSqlParserListener
-
- result []string
- err error
+func extractOmniUpdateColumns(stmt *oracleast.UpdateStmt) []string {
+ var result []string
+ if stmt == nil || stmt.SetClauses == nil {
+ return result
+ }
+ for _, item := range stmt.SetClauses.Items {
+ clause, ok := item.(*oracleast.SetClause)
+ if !ok {
+ continue
+ }
+ if clause.Column != nil {
+ result = append(result, clause.Column.Column)
+ }
+ for _, column := range omniColumnRefList(clause.Columns) {
+ result = append(result, column.Column)
+ }
+ }
+ return result
}
-func (l *updateElemListener) EnterColumn_based_update_set_clause(ctx *parser.Column_based_update_set_clauseContext) {
- if l.err != nil {
- return
+func omniColumnRefList(list *oracleast.List) []*oracleast.ColumnRef {
+ if list == nil {
+ return nil
}
- if ctx.Column_name() != nil {
- _, _, columnName, err := plsqlNormalizeColumnName("", ctx.Column_name())
- if err != nil {
- l.err = errors.Wrapf(err, "failed to normalize column name")
- return
- }
- l.result = append(l.result, columnName)
- return
- }
- if ctx.Paren_column_list() != nil {
- for _, column := range ctx.Paren_column_list().Column_list().AllColumn_name() {
- _, _, columnName, err := plsqlNormalizeColumnName("", column)
- if err != nil {
- l.err = errors.Wrapf(err, "failed to normalize column name")
- return
- }
- l.result = append(l.result, columnName)
+ result := make([]*oracleast.ColumnRef, 0, len(list.Items))
+ for _, item := range list.Items {
+ if column, ok := item.(*oracleast.ColumnRef); ok {
+ result = append(result, column)
}
}
+ return result
}
func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
@@ -345,6 +370,9 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (s
containsSourceTable := false
tables, err := prepareTransformation(sourceDatabase, list[i].Text)
if err != nil {
+ if err == errNoBackupableDML {
+ continue
+ }
return "", errors.Wrap(err, "failed to prepare transformation")
}
for _, table := range tables {
@@ -354,11 +382,14 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (s
}
}
if containsSourceTable {
- result = append(result, list[i].Text)
+ result = append(result, normalizeExtractedRestoreSQL(list[i].Text))
}
}
- // Statements include their leading whitespace and trailing semicolons.
- return strings.Join(result, ""), nil
+ return strings.Join(result, "\n"), nil
+}
+
+func normalizeExtractedRestoreSQL(statement string) string {
+ return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(statement), ";"))
}
func equalOrLess(a, b *storepb.Position) bool {
diff --git a/backend/plugin/parser/plsql/restore_test.go b/backend/plugin/parser/plsql/restore_test.go
index 94be8db12e..3b6a26cfaf 100644
--- a/backend/plugin/parser/plsql/restore_test.go
+++ b/backend/plugin/parser/plsql/restore_test.go
@@ -78,6 +78,124 @@ func TestRestore(t *testing.T) {
}
}
+func TestRestoreOmniBoundaryCases(t *testing.T) {
+ t.Run("multi-line update", func(t *testing.T) {
+ input := `UPDATE test
+SET c1 = 1
+WHERE c1 = 1;
+UPDATE test
+SET c1 = 2
+WHERE c1 = 2;`
+
+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(0, math.MaxInt32))
+ require.NoError(t, err)
+ require.Equal(t, `/*
+Original SQL:
+UPDATE test
+SET c1 = 1
+WHERE c1 = 1
+UPDATE test
+SET c1 = 2
+WHERE c1 = 2
+*/
+MERGE INTO "DB"."TEST" t
+USING "bbarchive"."prefix_1_test" b
+ ON( t."A" = b."A")
+WHEN MATCHED THEN
+ UPDATE SET t."C1" = b."C1"
+WHEN NOT MATCHED THEN
+ INSERT ("A", "B", "C") VALUES (b."A", b."B", b."C");`, result)
+ })
+
+ t.Run("position range selects update", func(t *testing.T) {
+ input := `DELETE FROM test WHERE c1 = 1;
+UPDATE test SET c1 = 2 WHERE c1 = 2;`
+
+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(2, 2))
+ require.NoError(t, err)
+ require.Contains(t, result, "UPDATE test SET c1 = 2 WHERE c1 = 2")
+ require.NotContains(t, result, "DELETE FROM test WHERE c1 = 1")
+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)
+ })
+
+ t.Run("same-line multi statement", func(t *testing.T) {
+ input := `UPDATE test SET c1 = 1 WHERE c1 = 1; UPDATE test SET c1 = 2 WHERE c1 = 2;`
+
+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(0, math.MaxInt32))
+ require.NoError(t, err)
+ require.Contains(t, result, "UPDATE test SET c1 = 1 WHERE c1 = 1\nUPDATE test SET c1 = 2 WHERE c1 = 2")
+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)
+ })
+
+ t.Run("range skips non backupable statements", func(t *testing.T) {
+ input := `INSERT INTO test (a, b, c) VALUES (1, 1, 1);
+UPDATE test SET c1 = 2 WHERE c1 = 2;
+CREATE TABLE unrelated (id NUMBER);`
+
+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(0, math.MaxInt32))
+ require.NoError(t, err)
+ require.Contains(t, result, "UPDATE test SET c1 = 2 WHERE c1 = 2")
+ require.NotContains(t, result, "INSERT INTO test")
+ require.NotContains(t, result, "CREATE TABLE unrelated")
+ require.Contains(t, result, `UPDATE SET t."C1" = b."C1"`)
+ })
+
+ t.Run("multi-line delete", func(t *testing.T) {
+ input := `DELETE FROM test
+WHERE c1 = 1;`
+
+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(0, math.MaxInt32))
+ require.NoError(t, err)
+ require.Equal(t, `/*
+Original SQL:
+DELETE FROM test
+WHERE c1 = 1
+*/
+INSERT INTO "DB"."TEST" ("A", "B", "C") SELECT "A", "B", "C" FROM "bbarchive"."prefix_1_test";`, result)
+ })
+
+ t.Run("no disjoint unique key", func(t *testing.T) {
+ input := `UPDATE test SET a = 1 WHERE c1 = 1;`
+
+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{
+ GetDatabaseMetadataFunc: fixedMockDatabaseMetadataGetter,
+ }, input, restoreBackupItem(0, math.MaxInt32))
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "no disjoint unique key found")
+ })
+}
+
+func restoreBackupItem(startLine, endLine int32) *store.PriorBackupDetail_Item {
+ return &store.PriorBackupDetail_Item{
+ SourceTable: &store.PriorBackupDetail_Item_Table{
+ Database: "instances/i1/databases/DB",
+ Table: "TEST",
+ },
+ TargetTable: &store.PriorBackupDetail_Item_Table{
+ Database: "instances/i1/databases/bbarchive",
+ Table: "prefix_1_test",
+ },
+ StartPosition: &store.Position{
+ Line: startLine,
+ Column: 0,
+ },
+ EndPosition: &store.Position{
+ Line: endLine,
+ Column: math.MaxInt32,
+ },
+ }
+}
+
func fixedMockDatabaseMetadataGetter(_ context.Context, _ string, database string) (string, *model.DatabaseMetadata, error) {
return database, model.NewDatabaseMetadata(&store.DatabaseSchemaMetadata{
Name: database,
diff --git a/backend/plugin/parser/plsql/util.go b/backend/plugin/parser/plsql/util.go
index 198df54d4e..e61fc70c1e 100644
--- a/backend/plugin/parser/plsql/util.go
+++ b/backend/plugin/parser/plsql/util.go
@@ -1,10 +1,12 @@
package plsql
import (
+ "github.com/antlr4-go/antlr/v4"
"github.com/bytebase/parser/plsql"
"github.com/pkg/errors"
)
+//nolint:unused
func plsqlNormalizeColumnName(currentSchema string, ctx plsql.IColumn_nameContext) (string, string, string, error) {
var buf []string
buf = append(buf, NormalizeIdentifierContext(ctx.Identifier()))
@@ -64,3 +66,17 @@ func NormalizeQuotedString(ctx plsql.IQuoted_stringContext) string {
raw := ctx.GetText()
return raw[1 : len(raw)-1]
}
+
+func IsTopLevelStatement(ctx antlr.Tree) bool {
+ if ctx == nil {
+ return true
+ }
+ switch ctx := ctx.(type) {
+ case *plsql.Unit_statementContext, *plsql.Sql_scriptContext:
+ return true
+ case *plsql.Data_manipulation_language_statementsContext:
+ return IsTopLevelStatement(ctx.GetParent())
+ default:
+ return false
+ }
+}