Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/mysql): migrate diagnose, query validation, and resource change to omni AST [BYT-9113, BYT-9114, BYT-9118] (#19844)
rebelice8a0921a6aaba
diff --git a/backend/plugin/parser/mysql/diagnose.go b/backend/plugin/parser/mysql/diagnose.goindex 66e682e809..a10d33f0c3 100644--- a/backend/plugin/parser/mysql/diagnose.go+++ b/backend/plugin/parser/mysql/diagnose.go@@ -2,11 +2,11 @@ package mysqlimport ("context"+ "errors""strings""unicode"- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/mysql"+ mysqlomniparser "github.com/bytebase/omni/mysql/parser"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -36,36 +36,24 @@ func parseMySQLStatement(statement string) *base.SyntaxError {// for the last statement in the script.statement += ";"}- inputStream := antlr.NewInputStream(statement)- lexer := parser.NewMySQLLexer(inputStream)- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- p := parser.NewMySQLParser(stream)- startPosition := &storepb.Position{Line: 1}- lexerErrorListener := &base.ParseErrorListener{- Statement: statement,- StartPosition: startPosition,+ _, err := ParseMySQLOmni(statement)+ if err == nil {+ return nil}- lexer.RemoveErrorListeners()- lexer.AddErrorListener(lexerErrorListener)- parserErrorListener := &base.ParseErrorListener{- Statement: statement,- StartPosition: startPosition,+ var parseErr *mysqlomniparser.ParseError+ if !errors.As(err, &parseErr) {+ return &base.SyntaxError{+ Position: &storepb.Position{Line: 1, Column: 1},+ Message: err.Error(),+ }}- p.RemoveErrorListeners()- p.AddErrorListener(parserErrorListener)- p.BuildParseTrees = false-- _ = p.Script()-- if lexerErrorListener.Err != nil {- return lexerErrorListener.Err- }-- if parserErrorListener.Err != nil {- return parserErrorListener.Err+ pos := ByteOffsetToRunePosition(statement, parseErr.Position)+ return &base.SyntaxError{+ Position: pos,+ Message: parseErr.Message,+ RawMessage: parseErr.Message,}- return nil}diff --git a/backend/plugin/parser/mysql/query.go b/backend/plugin/parser/mysql/query.goindex 6b91fa6ed7..a67f784e21 100644--- a/backend/plugin/parser/mysql/query.go+++ b/backend/plugin/parser/mysql/query.go@@ -1,8 +1,7 @@package mysqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/mysql"+ "github.com/bytebase/omni/mysql/ast"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -15,80 +14,31 @@ func init() {}// validateQuery validates the SQL statement for SQL editor.-// We validate the statement by following steps:-// 1. Remove all quoted text(quoted identifier, string literal) and comments from the statement.-// 2. Use regexp to check if the statement is a normal SELECT statement and EXPLAIN statement.-// 3. For CTE, use regexp to check if the statement has UPDATE, DELETE and INSERT statements.+// Only SELECT, EXPLAIN, SHOW, SET, and DESCRIBE are allowed in read-only mode.+// EXPLAIN ANALYZE is treated as non-read-only since it actually executes the query.func validateQuery(statement string) (bool, bool, error) {- trees, err := ParseMySQL(statement)+ stmts, err := ParseMySQLOmni(statement)if err != nil {return false, false, err}+hasExecute := falsereadOnly := true- for _, item := range trees {- l := &queryValidateListener{- valid: true,- hasExecute: false,- }- antlr.ParseTreeWalkerDefault.Walk(l, item.Tree)- if !l.valid {- return false, false, nil- }- if l.explainAnalyze {- readOnly = false- }- if l.hasExecute {+ for _, node := range stmts.Items {+ switch stmt := node.(type) {+ case *ast.SelectStmt:+ // SELECT is always allowed.+ case *ast.ExplainStmt:+ if stmt.Analyze {+ readOnly = false+ }+ case *ast.ShowStmt:+ // SHOW is always allowed.+ case *ast.SetStmt, *ast.SetPasswordStmt, *ast.SetDefaultRoleStmt, *ast.SetRoleStmt, *ast.SetResourceGroupStmt, *ast.SetTransactionStmt:hasExecute = true+ default:+ return false, false, nil}}return readOnly, !hasExecute, nil}--type queryValidateListener struct {- *parser.BaseMySQLParserListener-- valid bool- explainAnalyze bool- hasExecute bool-}--// EnterQuery is called when production query is entered.-func (l *queryValidateListener) EnterQuery(ctx *parser.QueryContext) {- if !l.valid {- return- }- if ctx.BeginWork() != nil {- l.valid = false- }-}--// EnterSimpleStatement is called when production simpleStatement is entered.-func (l *queryValidateListener) EnterSimpleStatement(ctx *parser.SimpleStatementContext) {- if !l.valid {- return- }- if ctx.SetStatement() != nil {- l.hasExecute = true- }- if ctx.SelectStatement() == nil && ctx.UtilityStatement() == nil && ctx.SetStatement() == nil && ctx.ShowStatement() == nil {- l.valid = false- }-}--// EnterUtilityStatement is called when production utilityStatement is entered.-func (l *queryValidateListener) EnterUtilityStatement(ctx *parser.UtilityStatementContext) {- if !l.valid {- return- }- if ctx.ExplainStatement() == nil && ctx.DescribeStatement() == nil {- l.valid = false- return- }- if ctx.ExplainStatement() != nil {- if ctx.ExplainStatement().ANALYZE_SYMBOL() != nil {- l.explainAnalyze = true- return- }- }-}diff --git a/backend/plugin/parser/mysql/query_test.go b/backend/plugin/parser/mysql/query_test.goindex 737eab6660..7cbcc898ae 100644--- a/backend/plugin/parser/mysql/query_test.go+++ b/backend/plugin/parser/mysql/query_test.go@@ -91,7 +91,6 @@ func TestValidateSQLForEditor(t *testing.T) {`,valid: false,gotAllQuery: false,- err: true,},{statement: "select * from t where a = 'klasjdfkljsa$tag$; -- lkjdlkfajslkdfj'",diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.goindex 3070f3b83d..71dad113fb 100644--- a/backend/plugin/parser/mysql/resource_change.go+++ b/backend/plugin/parser/mysql/resource_change.go@@ -1,8 +1,9 @@package mysqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/mysql"+ "strings"++ "github.com/bytebase/omni/mysql/ast""github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"@@ -19,357 +20,156 @@ func init() {base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)}-func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, statement string) (*base.ChangeSummary, error) {+func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {changedResources := model.NewChangedResources(dbMetadata)- l := &resourceChangedListener{- currentDatabase: currentDatabase,- statement: statement,- changedResources: changedResources,- }- for _, ast := range asts {- antlrAST, ok := base.GetANTLRAST(ast)- if !ok {- return nil, errors.New("expected ANTLR AST for MySQL")- }- l.reset()- antlr.ParseTreeWalkerDefault.Walk(l, antlrAST.Tree)- }- return &base.ChangeSummary{- ChangedResources: changedResources,- SampleDMLS: l.sampleDMLs,- DMLCount: l.dmlCount,- InsertCount: l.insertCount,- }, nil-}--type resourceChangedListener struct {- *parser.BaseMySQLParserListener+ var dmlCount, insertCount int+ var sampleDMLs []string- currentDatabase string- statement string-- changedResources *model.ChangedResources- sampleDMLs []string- dmlCount int- insertCount int-- // Internal data structure used temporarily.- text string-}--func (l *resourceChangedListener) reset() {- l.text = ""-}--func (l *resourceChangedListener) EnterQuery(ctx *parser.QueryContext) {- l.text = ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx)-}--// EnterCreateTable is called when production createTable is entered.-func (l *resourceChangedListener) EnterCreateTable(ctx *parser.CreateTableContext) {- database, table := NormalizeMySQLTableName(ctx.TableName())- if database == "" {- database = l.currentDatabase- }-- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )-}--// EnterDropTable is called when production dropTable is entered.-func (l *resourceChangedListener) EnterDropTable(ctx *parser.DropTableContext) {- for _, table := range ctx.TableRefList().AllTableRef() {- database, table := NormalizeMySQLTableRef(table)- if database == "" {- database = l.currentDatabase+ for _, unifiedAST := range asts {+ omniAST, ok := unifiedAST.(*OmniAST)+ if !ok {+ return nil, errors.New("expected OmniAST for MySQL")+ }+ if omniAST.Node == nil {+ continue}- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- true,- )- }-}--// EnterAlterTable is called when production alterTable is entered.-func (l *resourceChangedListener) EnterAlterTable(ctx *parser.AlterTableContext) {- database, table := NormalizeMySQLTableRef(ctx.TableRef())- if database == "" {- database = l.currentDatabase- }-- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- true,- )-}--// EnterRenameTableStatement is called when production renameTableStatement is entered.-func (l *resourceChangedListener) EnterRenameTableStatement(ctx *parser.RenameTableStatementContext) {- for _, pair := range ctx.AllRenamePair() {- {- database, table := NormalizeMySQLTableRef(pair.TableRef())- if database == "" {- database = l.currentDatabase+ switch n := omniAST.Node.(type) {+ case *ast.CreateTableStmt:+ if n.Table != nil {+ db, table := omniTableRef(n.Table, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)}- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )- }- {- database, table := NormalizeMySQLTableName(pair.TableName())- if database == "" {- database = l.currentDatabase+ case *ast.DropTableStmt:+ for _, ref := range n.Tables {+ db, table := omniTableRef(ref, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, true)}- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )- }- }-}--func (l *resourceChangedListener) EnterCreateIndex(ctx *parser.CreateIndexContext) {- if ctx.CreateIndexTarget() == nil || ctx.CreateIndexTarget().TableRef() == nil {- return- }- database, table := NormalizeMySQLTableRef(ctx.CreateIndexTarget().TableRef())- if database == "" {- database = l.currentDatabase- }-- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )-}--func (l *resourceChangedListener) EnterDropIndex(ctx *parser.DropIndexContext) {- if ctx.TableRef() == nil {- return- }-- database, table := NormalizeMySQLTableRef(ctx.TableRef())- if database == "" {- database = l.currentDatabase- }+ case *ast.AlterTableStmt:+ if n.Table != nil {+ db, table := omniTableRef(n.Table, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, true)+ }- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )-}+ case *ast.RenameTableStmt:+ for _, pair := range n.Pairs {+ if pair.Old != nil {+ db, table := omniTableRef(pair.Old, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)+ }+ if pair.New != nil {+ db, table := omniTableRef(pair.New, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)+ }+ }-func (l *resourceChangedListener) EnterInsertStatement(ctx *parser.InsertStatementContext) {- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {- return- }- if ctx.TableRef() == nil {- return- }+ case *ast.CreateIndexStmt:+ if n.Table != nil {+ db, table := omniTableRef(n.Table, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)+ }- database, table := NormalizeMySQLTableRef(ctx.TableRef())- if database == "" {- database = l.currentDatabase- }+ case *ast.DropIndexStmt:+ if n.Table != nil {+ db, table := omniTableRef(n.Table, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)+ }- l.changedResources.AddTable(- database,- "",- &storepb.ChangedResourceTable{- Name: table,- },- false,- )+ case *ast.InsertStmt:+ if n.Table != nil {+ db, table := omniTableRef(n.Table, currentDatabase)+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, false)+ }+ if len(n.Values) > 0 {+ insertCount += len(n.Values)+ continue+ }+ dmlCount+++ if len(sampleDMLs) < common.MaximumLintExplainSize {+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))+ }- if ctx.InsertFromConstructor() != nil && ctx.InsertFromConstructor().InsertValues() != nil && ctx.InsertFromConstructor().InsertValues().ValueList() != nil {- l.insertCount += len(ctx.InsertFromConstructor().InsertValues().ValueList().AllValues())- return- }+ case *ast.UpdateStmt:+ for _, resource := range extractTableExprs(n.Tables, currentDatabase) {+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)+ }+ dmlCount+++ if len(sampleDMLs) < common.MaximumLintExplainSize {+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))+ }- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)- }-}+ case *ast.DeleteStmt:+ for _, resource := range extractTableExprs(n.Tables, currentDatabase) {+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)+ }+ for _, resource := range extractTableExprs(n.Using, currentDatabase) {+ changedResources.AddTable(resource.Database, "", &storepb.ChangedResourceTable{Name: resource.Table}, false)+ }+ dmlCount+++ if len(sampleDMLs) < common.MaximumLintExplainSize {+ sampleDMLs = append(sampleDMLs, omniStatementText(omniAST))+ }-func (l *resourceChangedListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {- return- }- for _, tableRefCtx := range ctx.TableReferenceList().AllTableReference() {- resources := l.extractTableReference(tableRefCtx)- for _, resource := range resources {- l.changedResources.AddTable(- resource.Database,- "",- &storepb.ChangedResourceTable{- Name: resource.Table,- },- false,- )+ default:}}- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)- }+ return &base.ChangeSummary{+ ChangedResources: changedResources,+ SampleDMLS: sampleDMLs,+ DMLCount: dmlCount,+ InsertCount: insertCount,+ }, nil}-func (l *resourceChangedListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {- if !IsTopMySQLRule(&ctx.BaseParserRuleContext) {- return- }- var allResources []base.SchemaResource- if ctx.TableRef() != nil {- resources := l.extractTableRef(ctx.TableRef())- allResources = append(allResources, resources...)- }- if ctx.TableReferenceList() != nil {- resources := l.extractTableReferenceList(ctx.TableReferenceList())- allResources = append(allResources, resources...)- }-- for _, resource := range allResources {- l.changedResources.AddTable(- resource.Database,- "",- &storepb.ChangedResourceTable{- Name: resource.Table,- },- false,- )+// omniTableRef extracts database and table name from an omni TableRef.+func omniTableRef(ref *ast.TableRef, defaultDB string) (string, string) {+ if ref == nil {+ return defaultDB, ""}-- // Track DMLs.- l.dmlCount++- if len(l.sampleDMLs) < common.MaximumLintExplainSize {- l.sampleDMLs = append(l.sampleDMLs, l.text)+ db := ref.Schema+ if db == "" {+ db = defaultDB}+ return db, ref.Name}-func (l *resourceChangedListener) extractTableReference(ctx parser.ITableReferenceContext) []base.SchemaResource {- if ctx.TableFactor() == nil {- return nil- }- res := l.extractTableFactor(ctx.TableFactor())- for _, joinedTableCtx := range ctx.AllJoinedTable() {- tables := l.extractJoinedTable(joinedTableCtx)- res = append(res, tables...)+// extractTableExprs collects schema resources from a slice of TableExpr.+func extractTableExprs(exprs []ast.TableExpr, defaultDB string) []base.SchemaResource {+ var result []base.SchemaResource+ for _, expr := range exprs {+ result = append(result, extractTableExpr(expr, defaultDB)...)}-- return res+ return result}-func (l *resourceChangedListener) extractTableRef(ctx parser.ITableRefContext) []base.SchemaResource {- if ctx == nil {+// extractTableExpr recursively extracts schema resources from a single TableExpr.+func extractTableExpr(expr ast.TableExpr, defaultDB string) []base.SchemaResource {+ if expr == nil {return nil}- resource := base.SchemaResource{- Database: l.currentDatabase,- }- db, table := NormalizeMySQLTableRef(ctx)- if db != "" {- resource.Database = db- }- resource.Table = table-- return []base.SchemaResource{resource}-}--func (l *resourceChangedListener) extractTableReferenceList(ctx parser.ITableReferenceListContext) []base.SchemaResource {- var res []base.SchemaResource- for _, tableRefCtx := range ctx.AllTableReference() {- tables := l.extractTableReference(tableRefCtx)- res = append(res, tables...)- }- return res-}--func (l *resourceChangedListener) extractTableReferenceListParens(ctx parser.ITableReferenceListParensContext) []base.SchemaResource {- if ctx.TableReferenceList() != nil {- return l.extractTableReferenceList(ctx.TableReferenceList())- }- if ctx.TableReferenceListParens() != nil {- return l.extractTableReferenceListParens(ctx.TableReferenceListParens())- }- return nil-}--func (l *resourceChangedListener) extractTableFactor(ctx parser.ITableFactorContext) []base.SchemaResource {- switch {- case ctx.SingleTable() != nil:- return l.extractSingleTable(ctx.SingleTable())- case ctx.SingleTableParens() != nil:- return l.extractSingleTableParens(ctx.SingleTableParens())- case ctx.DerivedTable() != nil:- return nil- case ctx.TableReferenceListParens() != nil:- return l.extractTableReferenceListParens(ctx.TableReferenceListParens())- case ctx.TableFunction() != nil:- return nil+ switch n := expr.(type) {+ case *ast.TableRef:+ db, table := omniTableRef(n, defaultDB)+ return []base.SchemaResource{{Database: db, Table: table}}+ case *ast.JoinClause:+ var res []base.SchemaResource+ res = append(res, extractTableExpr(n.Left, defaultDB)...)+ res = append(res, extractTableExpr(n.Right, defaultDB)...)+ return resdefault:return nil}}-func (l *resourceChangedListener) extractSingleTable(ctx parser.ISingleTableContext) []base.SchemaResource {- return l.extractTableRef(ctx.TableRef())-}--func (l *resourceChangedListener) extractSingleTableParens(ctx parser.ISingleTableParensContext) []base.SchemaResource {- if ctx.SingleTable() != nil {- return l.extractSingleTable(ctx.SingleTable())- }- if ctx.SingleTableParens() != nil {- return l.extractSingleTableParens(ctx.SingleTableParens())- }- return nil-}--func (l *resourceChangedListener) extractJoinedTable(ctx parser.IJoinedTableContext) []base.SchemaResource {- if ctx.TableFactor() != nil {- return l.extractTableFactor(ctx.TableFactor())- }- if ctx.TableReference() != nil {- return l.extractTableReference(ctx.TableReference())+// omniStatementText returns the text of a statement, ensuring it ends with a semicolon.+func omniStatementText(a *OmniAST) string {+ text := strings.TrimSpace(a.Text)+ if !strings.HasSuffix(text, ";") {+ text += ";"}- return nil+ return text}