Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser/mysql): migrate diagnose, query validation, and resource change to omni AST [BYT-9113, BYT-9114, BYT-9118] (#19844)

rebelice8a0921a6aaba
diff --git a/backend/plugin/parser/mysql/diagnose.go b/backend/plugin/parser/mysql/diagnose.go
index 66e682e809..a10d33f0c3 100644
--- a/backend/plugin/parser/mysql/diagnose.go
+++ b/backend/plugin/parser/mysql/diagnose.go
@@ -2,11 +2,11 @@ package mysql
import (
"context"
+ "errors"
"strings"
"unicode"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/mysql"
+ mysqlomniparser "github.com/bytebase/omni/mysql/parser"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -36,36 +36,24 @@ func parseMySQLStatement(statement string) *base.SyntaxError {
// for the last statement in the script.
statement += ";"
}
- inputStream := antlr.NewInputStream(statement)
- lexer := parser.NewMySQLLexer(inputStream)
- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
- p := parser.NewMySQLParser(stream)
- startPosition := &storepb.Position{Line: 1}
- lexerErrorListener := &base.ParseErrorListener{
- Statement: statement,
- StartPosition: startPosition,
+ _, err := ParseMySQLOmni(statement)
+ if err == nil {
+ return nil
}
- lexer.RemoveErrorListeners()
- lexer.AddErrorListener(lexerErrorListener)
- parserErrorListener := &base.ParseErrorListener{
- Statement: statement,
- StartPosition: startPosition,
+ var parseErr *mysqlomniparser.ParseError
+ if !errors.As(err, &parseErr) {
+ return &base.SyntaxError{
+ Position: &storepb.Position{Line: 1, Column: 1},
+ Message: err.Error(),
+ }
}
- p.RemoveErrorListeners()
- p.AddErrorListener(parserErrorListener)
- p.BuildParseTrees = false
-
- _ = p.Script()
-
- if lexerErrorListener.Err != nil {
- return lexerErrorListener.Err
- }
-
- if parserErrorListener.Err != nil {
- return parserErrorListener.Err
+ pos := ByteOffsetToRunePosition(statement, parseErr.Position)
+ return &base.SyntaxError{
+ Position: pos,
+ Message: parseErr.Message,
+ RawMessage: parseErr.Message,
}
- return nil
}
diff --git a/backend/plugin/parser/mysql/query.go b/backend/plugin/parser/mysql/query.go
index 6b91fa6ed7..a67f784e21 100644
--- a/backend/plugin/parser/mysql/query.go
+++ b/backend/plugin/parser/mysql/query.go
@@ -1,8 +1,7 @@
package mysql
import (
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/mysql"
+ "github.com/bytebase/omni/mysql/ast"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -15,80 +14,31 @@ func init() {
}
// validateQuery validates the SQL statement for SQL editor.
-// We validate the statement by following steps:
-// 1. Remove all quoted text(quoted identifier, string literal) and comments from the statement.
-// 2. Use regexp to check if the statement is a normal SELECT statement and EXPLAIN statement.
-// 3. For CTE, use regexp to check if the statement has UPDATE, DELETE and INSERT statements.
+// Only SELECT, EXPLAIN, SHOW, SET, and DESCRIBE are allowed in read-only mode.
+// EXPLAIN ANALYZE is treated as non-read-only since it actually executes the query.
func validateQuery(statement string) (bool, bool, error) {
- trees, err := ParseMySQL(statement)
+ stmts, err := ParseMySQLOmni(statement)
if err != nil {
return false, false, err
}
+
hasExecute := false
readOnly := true
- for _, item := range trees {
- l := &queryValidateListener{
- valid: true,
- hasExecute: false,
- }
- antlr.ParseTreeWalkerDefault.Walk(l, item.Tree)
- if !l.valid {
- return false, false, nil
- }
- if l.explainAnalyze {
- readOnly = false
- }
- if l.hasExecute {
+ for _, node := range stmts.Items {
+ switch stmt := node.(type) {
+ case *ast.SelectStmt:
+ // SELECT is always allowed.
+ case *ast.ExplainStmt:
+ if stmt.Analyze {
+ readOnly = false
+ }
+ case *ast.ShowStmt:
+ // SHOW is always allowed.
+ case *ast.SetStmt, *ast.SetPasswordStmt, *ast.SetDefaultRoleStmt, *ast.SetRoleStmt, *ast.SetResourceGroupStmt, *ast.SetTransactionStmt:
hasExecute = true
+ default:
+ return false, false, nil
}
}
return readOnly, !hasExecute, nil
}
-
-type queryValidateListener struct {
- *parser.BaseMySQLParserListener
-
- valid bool
- explainAnalyze bool
- hasExecute bool
-}
-
-// EnterQuery is called when production query is entered.
-func (l *queryValidateListener) EnterQuery(ctx *parser.QueryContext) {
- if !l.valid {
- return
- }
- if ctx.BeginWork() != nil {
- l.valid = false
- }
-}
-
-// EnterSimpleStatement is called when production simpleStatement is entered.
-func (l *queryValidateListener) EnterSimpleStatement(ctx *parser.SimpleStatementContext) {
- if !l.valid {
- return
- }
- if ctx.SetStatement() != nil {
- l.hasExecute = true
- }
- if ctx.SelectStatement() == nil && ctx.UtilityStatement() == nil && ctx.SetStatement() == nil && ctx.ShowStatement() == nil {
- l.valid = false
- }
-}
-
-// EnterUtilityStatement is called when production utilityStatement is entered.
-func (l *queryValidateListener) EnterUtilityStatement(ctx *parser.UtilityStatementContext) {
- if !l.valid {
- return
- }
- if ctx.ExplainStatement() == nil && ctx.DescribeStatement() == nil {
- l.valid = false
- return
- }
- if ctx.ExplainStatement() != nil {
- if ctx.ExplainStatement().ANALYZE_SYMBOL() != nil {
- l.explainAnalyze = true
- return
- }
- }
-}
diff --git a/backend/plugin/parser/mysql/query_test.go b/backend/plugin/parser/mysql/query_test.go
index 737eab6660..7cbcc898ae 100644
--- a/backend/plugin/parser/mysql/query_test.go
+++ b/backend/plugin/parser/mysql/query_test.go
@@ -91,7 +91,6 @@ func TestValidateSQLForEditor(t *testing.T) {
`,
valid: false,
gotAllQuery: false,
- err: true,
},
{
statement: "select * from t where a = 'klasjdfkljsa$tag$; -- lkjdlkfajslkdfj'",
diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.go
index 3070f3b83d..71dad113fb 100644
--- a/backend/plugin/parser/mysql/resource_change.go
+++ b/backend/plugin/parser/mysql/resource_change.go
@@ -1,8 +1,9 @@
package mysql
import (
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/mysql"
+ "strings"
+
+ "github.com/bytebase/omni/mysql/ast"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
@@ -19,357 +20,156 @@ func init() {
base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)
}
-func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {
+func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
changedResources := model.NewChangedResources(dbMetadata)
- l := &resourceChangedListener{
- currentDatabase: currentDatabase,
- statement: statement,
- changedResources: changedResources,
- }
- for _, ast := range asts {
- antlrAST, ok := base.GetANTLRAST(ast)
- if !ok {
- return nil, errors.New("expected ANTLR AST for MySQL")
- }
- l.reset()
- antlr.ParseTreeWalkerDefault.Walk(l, antlrAST.Tree)
- }
- return &base.ChangeSummary{
- ChangedResources: changedResources,
- SampleDMLS: l.sampleDMLs,
- DMLCount: l.dmlCount,
- InsertCount: l.insertCount,
- }, nil
-}
-
-type resourceChangedListener struct {
- *parser.BaseMySQLParserListener
+ var dmlCount, insertCount int
+ var sampleDMLs []string
- currentDatabase string
- statement string
-
- changedResources *model.ChangedResources
- sampleDMLs []string
- dmlCount int
- insertCount int
-
- // Internal data structure used temporarily.
- text string
-}
-
-func (l *resourceChangedListener) reset() {
- l.text = ""
-}
-
-func (l *resourceChangedListener) EnterQuery(ctx *parser.QueryContext) {
- l.text = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)
-}
-
-// EnterCreateTable is called when production createTable is entered.
-func (l *resourceChangedListener) EnterCreateTable(ctx *parser.CreateTableContext) {
- database, table := NormalizeMySQLTableName(ctx.TableName())
- if database == "" {
- database = l.currentDatabase
- }
-
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
-}
-
-// EnterDropTable is called when production dropTable is entered.
-func (l *resourceChangedListener) EnterDropTable(ctx *parser.DropTableContext) {
- for _, table := range ctx.TableRefList().AllTableRef() {
- database, table := NormalizeMySQLTableRef(table)
- if database == "" {
- database = l.currentDatabase
+ for _, unifiedAST := range asts {
+ omniAST, ok := unifiedAST.(*OmniAST)
+ if !ok {
+ return nil, errors.New("expected OmniAST for MySQL")
+ }
+ if omniAST.Node == nil {
+ continue
}
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- true,
- )
- }
-}
-
-// EnterAlterTable is called when production alterTable is entered.
-func (l *resourceChangedListener) EnterAlterTable(ctx *parser.AlterTableContext) {
- database, table := NormalizeMySQLTableRef(ctx.TableRef())
- if database == "" {
- database = l.currentDatabase
- }
-
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- true,
- )
-}
-
-// EnterRenameTableStatement is called when production renameTableStatement is entered.
-func (l *resourceChangedListener) EnterRenameTableStatement(ctx *parser.RenameTableStatementContext) {
- for _, pair := range ctx.AllRenamePair() {
- {
- database, table := NormalizeMySQLTableRef(pair.TableRef())
- if database == "" {
- database = l.currentDatabase
+ switch n := omniAST.Node.(type) {
+ case *ast.CreateTableStmt:
+ if n.Table != nil {
+ db, table := omniTableRef(n.Table, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
}
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
- }
- {
- database, table := NormalizeMySQLTableName(pair.TableName())
- if database == "" {
- database = l.currentDatabase
+ case *ast.DropTableStmt:
+ for _, ref := range n.Tables {
+ db, table := omniTableRef(ref, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, true)
}
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
- }
- }
-}
-
-func (l *resourceChangedListener) EnterCreateIndex(ctx *parser.CreateIndexContext) {
- if ctx.CreateIndexTarget() == nil || ctx.CreateIndexTarget().TableRef() == nil {
- return
- }
- database, table := NormalizeMySQLTableRef(ctx.CreateIndexTarget().TableRef())
- if database == "" {
- database = l.currentDatabase
- }
-
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
-}
-
-func (l *resourceChangedListener) EnterDropIndex(ctx *parser.DropIndexContext) {
- if ctx.TableRef() == nil {
- return
- }
-
- database, table := NormalizeMySQLTableRef(ctx.TableRef())
- if database == "" {
- database = l.currentDatabase
- }
+ case *ast.AlterTableStmt:
+ if n.Table != nil {
+ db, table := omniTableRef(n.Table, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, true)
+ }
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
-}
+ case *ast.RenameTableStmt:
+ for _, pair := range n.Pairs {
+ if pair.Old != nil {
+ db, table := omniTableRef(pair.Old, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ if pair.New != nil {
+ db, table := omniTableRef(pair.New, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ }
-func (l *resourceChangedListener) EnterInsertStatement(ctx *parser.InsertStatementContext) {
- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {
- return
- }
- if ctx.TableRef() == nil {
- return
- }
+ case *ast.CreateIndexStmt:
+ if n.Table != nil {
+ db, table := omniTableRef(n.Table, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
+ }
- database, table := NormalizeMySQLTableRef(ctx.TableRef())
- if database == "" {
- database = l.currentDatabase
- }
+ case *ast.DropIndexStmt:
+ if n.Table != nil {
+ db, table := omniTableRef(n.Table, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
+ }
- l.changedResources.AddTable(
- database,
- "",
- &storepb.ChangedResourceTable{
- Name: table,
- },
- false,
- )
+ case *ast.InsertStmt:
+ if n.Table != nil {
+ db, table := omniTableRef(n.Table, currentDatabase)
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)
+ }
+ if len(n.Values) > 0 {
+ insertCount += len(n.Values)
+ continue
+ }
+ dmlCount++
+ if len(sampleDMLs) < common.MaximumLintExplainSize {
+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
+ }
- if ctx.InsertFromConstructor() != nil && ctx.InsertFromConstructor().InsertValues() != nil && ctx.InsertFromConstructor().InsertValues().ValueList() != nil {
- l.insertCount += len(ctx.InsertFromConstructor().InsertValues().ValueList().AllValues())
- return
- }
+ case *ast.UpdateStmt:
+ for _, resource := range extractTableExprs(n.Tables, currentDatabase) {
+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)
+ }
+ dmlCount++
+ if len(sampleDMLs) < common.MaximumLintExplainSize {
+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
+ }
- // Track DMLs.
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
- }
-}
+ case *ast.DeleteStmt:
+ for _, resource := range extractTableExprs(n.Tables, currentDatabase) {
+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)
+ }
+ for _, resource := range extractTableExprs(n.Using, currentDatabase) {
+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)
+ }
+ dmlCount++
+ if len(sampleDMLs) < common.MaximumLintExplainSize {
+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))
+ }
-func (l *resourceChangedListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {
- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {
- return
- }
- for _, tableRefCtx := range ctx.TableReferenceList().AllTableReference() {
- resources := l.extractTableReference(tableRefCtx)
- for _, resource := range resources {
- l.changedResources.AddTable(
- resource.Database,
- "",
- &storepb.ChangedResourceTable{
- Name: resource.Table,
- },
- false,
- )
+ default:
}
}
- // Track DMLs.
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
- }
+ return &base.ChangeSummary{
+ ChangedResources: changedResources,
+ SampleDMLS: sampleDMLs,
+ DMLCount: dmlCount,
+ InsertCount: insertCount,
+ }, nil
}
-func (l *resourceChangedListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {
- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {
- return
- }
- var allResources []base.SchemaResource
- if ctx.TableRef() != nil {
- resources := l.extractTableRef(ctx.TableRef())
- allResources = append(allResources, resources...)
- }
- if ctx.TableReferenceList() != nil {
- resources := l.extractTableReferenceList(ctx.TableReferenceList())
- allResources = append(allResources, resources...)
- }
-
- for _, resource := range allResources {
- l.changedResources.AddTable(
- resource.Database,
- "",
- &storepb.ChangedResourceTable{
- Name: resource.Table,
- },
- false,
- )
+// omniTableRef extracts database and table name from an omni TableRef.
+func omniTableRef(ref *ast.TableRef, defaultDB string) (string, string) {
+ if ref == nil {
+ return defaultDB, ""
}
-
- // Track DMLs.
- l.dmlCount++
- if len(l.sampleDMLs) < common.MaximumLintExplainSize {
- l.sampleDMLs = append(l.sampleDMLs, l.text)
+ db := ref.Schema
+ if db == "" {
+ db = defaultDB
}
+ return db, ref.Name
}
-func (l *resourceChangedListener) extractTableReference(ctx parser.ITableReferenceContext) []base.SchemaResource {
- if ctx.TableFactor() == nil {
- return nil
- }
- res := l.extractTableFactor(ctx.TableFactor())
- for _, joinedTableCtx := range ctx.AllJoinedTable() {
- tables := l.extractJoinedTable(joinedTableCtx)
- res = append(res, tables...)
+// extractTableExprs collects schema resources from a slice of TableExpr.
+func extractTableExprs(exprs []ast.TableExpr, defaultDB string) []base.SchemaResource {
+ var result []base.SchemaResource
+ for _, expr := range exprs {
+ result = append(result, extractTableExpr(expr, defaultDB)...)
}
-
- return res
+ return result
}
-func (l *resourceChangedListener) extractTableRef(ctx parser.ITableRefContext) []base.SchemaResource {
- if ctx == nil {
+// extractTableExpr recursively extracts schema resources from a single TableExpr.
+func extractTableExpr(expr ast.TableExpr, defaultDB string) []base.SchemaResource {
+ if expr == nil {
return nil
}
- resource := base.SchemaResource{
- Database: l.currentDatabase,
- }
- db, table := NormalizeMySQLTableRef(ctx)
- if db != "" {
- resource.Database = db
- }
- resource.Table = table
-
- return []base.SchemaResource{resource}
-}
-
-func (l *resourceChangedListener) extractTableReferenceList(ctx parser.ITableReferenceListContext) []base.SchemaResource {
- var res []base.SchemaResource
- for _, tableRefCtx := range ctx.AllTableReference() {
- tables := l.extractTableReference(tableRefCtx)
- res = append(res, tables...)
- }
- return res
-}
-
-func (l *resourceChangedListener) extractTableReferenceListParens(ctx parser.ITableReferenceListParensContext) []base.SchemaResource {
- if ctx.TableReferenceList() != nil {
- return l.extractTableReferenceList(ctx.TableReferenceList())
- }
- if ctx.TableReferenceListParens() != nil {
- return l.extractTableReferenceListParens(ctx.TableReferenceListParens())
- }
- return nil
-}
-
-func (l *resourceChangedListener) extractTableFactor(ctx parser.ITableFactorContext) []base.SchemaResource {
- switch {
- case ctx.SingleTable() != nil:
- return l.extractSingleTable(ctx.SingleTable())
- case ctx.SingleTableParens() != nil:
- return l.extractSingleTableParens(ctx.SingleTableParens())
- case ctx.DerivedTable() != nil:
- return nil
- case ctx.TableReferenceListParens() != nil:
- return l.extractTableReferenceListParens(ctx.TableReferenceListParens())
- case ctx.TableFunction() != nil:
- return nil
+ switch n := expr.(type) {
+ case *ast.TableRef:
+ db, table := omniTableRef(n, defaultDB)
+ return []base.SchemaResource{{Database: db, Table: table}}
+ case *ast.JoinClause:
+ var res []base.SchemaResource
+ res = append(res, extractTableExpr(n.Left, defaultDB)...)
+ res = append(res, extractTableExpr(n.Right, defaultDB)...)
+ return res
default:
return nil
}
}
-func (l *resourceChangedListener) extractSingleTable(ctx parser.ISingleTableContext) []base.SchemaResource {
- return l.extractTableRef(ctx.TableRef())
-}
-
-func (l *resourceChangedListener) extractSingleTableParens(ctx parser.ISingleTableParensContext) []base.SchemaResource {
- if ctx.SingleTable() != nil {
- return l.extractSingleTable(ctx.SingleTable())
- }
- if ctx.SingleTableParens() != nil {
- return l.extractSingleTableParens(ctx.SingleTableParens())
- }
- return nil
-}
-
-func (l *resourceChangedListener) extractJoinedTable(ctx parser.IJoinedTableContext) []base.SchemaResource {
- if ctx.TableFactor() != nil {
- return l.extractTableFactor(ctx.TableFactor())
- }
- if ctx.TableReference() != nil {
- return l.extractTableReference(ctx.TableReference())
+// omniStatementText returns the text of a statement, ensuring it ends with a semicolon.
+func omniStatementText(a *OmniAST) string {
+ text := strings.TrimSpace(a.Text)
+ if !strings.HasSuffix(text, ";") {
+ text += ";"
}
- return nil
+ return text
}