Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: migrate PostgreSQL query validator to ANTLR parser (#17903)
h3n4l2c419f6993b2
diff --git a/backend/plugin/parser/pg/query.go b/backend/plugin/parser/pg/query.goindex 50fb2f095d..4836cca3b9 100644--- a/backend/plugin/parser/pg/query.go+++ b/backend/plugin/parser/pg/query.go@@ -1,126 +1,12 @@package pgimport (- "encoding/json"- "log/slog"- "regexp"- "strings"-- pgquery "github.com/pganalyze/pg_query_go/v6"- pgparser "github.com/pganalyze/pg_query_go/v6/parser"-- "github.com/bytebase/bytebase/backend/common"- "github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"- pgrawparser "github.com/bytebase/bytebase/backend/plugin/parser/pg/legacy"- "github.com/bytebase/bytebase/backend/plugin/parser/pg/legacy/ast")func init() {- base.RegisterQueryValidator(storepb.Engine_POSTGRES, validateQuery)+ base.RegisterQueryValidator(storepb.Engine_POSTGRES, validateQueryANTLR)// Redshift has its own implementation in the redshift package- base.RegisterQueryValidator(storepb.Engine_COCKROACHDB, validateQuery)-}--// validateQuery validates the SQL statement for SQL editor.-// Consider that the tokenizer cannot handle the dollar-sign($), so that we use pg_query_go to parse the statement.-// For EXPLAIN and normal SELECT statements, we can directly use regexp to check.-// For CTE, we need to parse the statement to JSON and check the JSON keys.-func validateQuery(statement string) (bool, bool, error) {- stmtList, err := pgrawparser.Parse(pgrawparser.ParseContext{}, statement)- if err != nil {- return false, false, convertToSyntaxError(statement, err)- }-- explainAnalyze := false- hasExecute := false- for _, stmt := range stmtList {- switch stmt := stmt.(type) {- case *ast.SelectStmt, *ast.VariableShowStmt:- case *ast.VariableSetStmt:- hasExecute = true- case *ast.ExplainStmt:- if stmt.Analyze {- // We only support analyze select, because analyze will actually execute the statement.- if _, ok := stmt.Statement.(*ast.SelectStmt); !ok {- return false, false, nil- }- explainAnalyze = true- }- default:- return false, false, nil- }- }-- // TODO(d): figure out whether this is still needed.- jsonText, err := pgquery.ParseToJSON(statement)- if err != nil {- slog.Debug("Failed to parse statement to JSON", slog.String("statement", statement), log.BBError(err))- return false, false, err- }-- formattedStr := strings.ToUpper(strings.TrimSpace(statement))-- cteRegex := regexp.MustCompile(`^WITH\s+?`)- if matchResult := cteRegex.MatchString(formattedStr); matchResult || explainAnalyze {- var jsonData map[string]any-- if err := json.Unmarshal([]byte(jsonText), &jsonData); err != nil {- slog.Debug("Failed to unmarshal JSON", slog.String("jsonText", jsonText), log.BBError(err))- return false, false, err- }-- dmlKeyList := []string{"InsertStmt", "UpdateStmt", "DeleteStmt"}-- ok := !keyExistsInJSONData(jsonData, dmlKeyList)- return ok, ok, nil- }-- return true, !hasExecute, nil-}--func keyExistsInJSONData(jsonData map[string]any, keyList []string) bool {- for _, key := range keyList {- if _, ok := jsonData[key]; ok {- return true- }- }-- for _, value := range jsonData {- switch v := value.(type) {- case map[string]any:- if keyExistsInJSONData(v, keyList) {- return true- }- case []any:- for _, item := range v {- if m, ok := item.(map[string]any); ok {- if keyExistsInJSONData(m, keyList) {- return true- }- }- }- }- }-- return false-}--func convertToSyntaxError(statement string, err error) *base.SyntaxError {- if pgErr, ok := err.(*pgparser.Error); ok {- position := common.ConvertPGParserErrorCursorPosToPosition(pgErr.Cursorpos, statement)- return &base.SyntaxError{- Position: position,- Message: pgErr.Message,- }- }-- return &base.SyntaxError{- Position: &storepb.Position{- Line: 0,- Column: 0,- },- Message: err.Error(),- }+ base.RegisterQueryValidator(storepb.Engine_COCKROACHDB, validateQueryANTLR)}diff --git a/backend/plugin/parser/pg/query_antlr.go b/backend/plugin/parser/pg/query_antlr.gonew file mode 100644index 0000000000..7afd43da09--- /dev/null+++ b/backend/plugin/parser/pg/query_antlr.go@@ -0,0 +1,302 @@+package pg++import (+ "github.com/antlr4-go/antlr/v4"+ parser "github.com/bytebase/parser/postgresql"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// validateQueryANTLR validates the SQL statement for SQL editor using ANTLR parser.+// Returns (isReadOnly, allQueriesReturnData, error)+//+// Validation rules:+// 1. Allow: SELECT statements+// 2. Allow: EXPLAIN statements (but not EXPLAIN ANALYZE for non-SELECT)+// 3. Allow: SHOW/SET statements (SET is considered executable)+// 4. Allow: CTEs with only SELECT (reject CTEs with INSERT/UPDATE/DELETE)+// 5. Reject: All other statements (DDL, DML except SELECT)+func validateQueryANTLR(statement string) (bool, bool, error) {+ // Parse with ANTLR+ result, err := ParsePostgreSQL(statement)+ if err != nil {+ // Return syntax error+ if syntaxErr, ok := err.(*base.SyntaxError); ok {+ return false, false, syntaxErr+ }+ return false, false, err+ }++ // Analyze the parse tree+ tree := result.Tree+ listener := &queryValidatorListener{+ hasExecute: false,+ explainAnalyze: false,+ hasCTE: false,+ hasInvalidStmt: false,+ validStmtCount: 0,+ }++ antlr.ParseTreeWalkerDefault.Walk(listener, tree)++ // If we found invalid statements, reject+ if listener.hasInvalidStmt {+ return false, false, nil+ }++ // Check for DML in CTEs or EXPLAIN ANALYZE+ // This is done in a second pass to detect DML statements anywhere in the tree+ if listener.hasCTE || listener.explainAnalyze {+ dmlDetector := &dmlDetectorListener{}+ antlr.ParseTreeWalkerDefault.Walk(dmlDetector, tree)++ if dmlDetector.hasDML {+ return false, false, nil+ }+ }++ // Return validation results+ // isReadOnly = true (all valid queries are read-only in this context)+ // allQueriesReturnData = !hasExecute (SET statements don't return data)+ return true, !listener.hasExecute, nil+}++// queryValidatorListener walks the ANTLR parse tree to validate query types+type queryValidatorListener struct {+ *parser.BasePostgreSQLParserListener++ hasExecute bool // True if there are SET statements+ explainAnalyze bool // True if there's EXPLAIN ANALYZE+ hasCTE bool // True if there's a WITH clause (CTE)+ hasInvalidStmt bool // True if there's an invalid statement type+ validStmtCount int // Count of valid statements encountered+}++// EnterSelectstmt detects CTEs and marks statement as valid+func (l *queryValidatorListener) EnterSelectstmt(ctx *parser.SelectstmtContext) {+ // Only count top-level SELECT statements+ if isTopLevelStmt(ctx) {+ l.validStmtCount+++ }++ // Check if this SELECT has a WITH clause (CTE)+ // WITH clause is in Select_no_parens+ if ctx.Select_no_parens() != nil {+ selectNoParens := ctx.Select_no_parens()+ if selectNoParens.With_clause() != nil {+ l.hasCTE = true+ }+ }+}++// EnterExplainstmt handles EXPLAIN statements+func (l *queryValidatorListener) EnterExplainstmt(ctx *parser.ExplainstmtContext) {+ if !isTopLevelStmt(ctx) {+ return+ }++ l.validStmtCount++++ // Check if it's EXPLAIN ANALYZE+ if l.isExplainAnalyze(ctx) {+ l.explainAnalyze = true+ // EXPLAIN ANALYZE is only valid for SELECT+ // Check the explained statement+ if ctx.Explainablestmt() != nil {+ explainableStmt := ctx.Explainablestmt()+ // If it's not a SELECT statement, mark as invalid+ if explainableStmt.Selectstmt() == nil {+ l.hasInvalidStmt = true+ }+ }+ }+}++// EnterVariablesetstmt handles SET statements+func (l *queryValidatorListener) EnterVariablesetstmt(ctx *parser.VariablesetstmtContext) {+ if !isTopLevelStmt(ctx) {+ return+ }++ l.validStmtCount+++ l.hasExecute = true+}++// EnterVariableshowstmt handles SHOW statements+func (l *queryValidatorListener) EnterVariableshowstmt(ctx *parser.VariableshowstmtContext) {+ if !isTopLevelStmt(ctx) {+ return+ }++ l.validStmtCount+++}++// Mark DML and DDL statements as invalid (only SELECT, EXPLAIN, SET, SHOW are allowed for SQL editor)+// DML should be detected at top level only, as they may appear in CTEs (which we check separately)+// DDL should always be rejected++// DDL statement handlers - always mark as invalid++// EnterCreatestmt detects CREATE statements (not allowed)+func (l *queryValidatorListener) EnterCreatestmt(ctx *parser.CreatestmtContext) {+ if isTopLevelStmt(ctx) {+ l.hasInvalidStmt = true+ }+}++// EnterAltertablestmt detects ALTER TABLE statements (not allowed)+func (l *queryValidatorListener) EnterAltertablestmt(ctx *parser.AltertablestmtContext) {+ if isTopLevelStmt(ctx) {+ l.hasInvalidStmt = true+ }+}++// EnterDropstmt detects DROP statements (not allowed)+func (l *queryValidatorListener) EnterDropstmt(ctx *parser.DropstmtContext) {+ if isTopLevelStmt(ctx) {+ l.hasInvalidStmt = true+ }+}++// EnterTruncatestmt detects TRUNCATE statements (not allowed)+func (l *queryValidatorListener) EnterTruncatestmt(ctx *parser.TruncatestmtContext) {+ if isTopLevelStmt(ctx) {+ l.hasInvalidStmt = true+ }+}++// DML statement handlers - mark as invalid at top level or inside EXPLAIN++// EnterInsertstmt detects INSERT statements (not allowed at top level)+func (l *queryValidatorListener) EnterInsertstmt(ctx *parser.InsertstmtContext) {+ // Check if this is at top level or inside EXPLAIN+ parent := ctx.GetParent()+ for parent != nil {+ switch parent.(type) {+ case *parser.ExplainstmtContext:+ // INSERT inside EXPLAIN - mark as invalid+ l.hasInvalidStmt = true+ return+ case *parser.RootContext, *parser.StmtblockContext, *parser.StmtmultiContext, *parser.StmtContext, *parser.ExplainablestmtContext:+ // Continue walking up - ExplainablestmtContext is the wrapper for statements in EXPLAIN+ parent = parent.GetParent()+ default:+ // Hit some other context - this is nested (e.g., in CTE, subquery)+ // We'll handle CTEs separately+ return+ }+ }+ // Reached root - this is a top-level INSERT+ l.hasInvalidStmt = true+}++// EnterUpdatestmt detects UPDATE statements (not allowed at top level)+func (l *queryValidatorListener) EnterUpdatestmt(ctx *parser.UpdatestmtContext) {+ // Check if this is at top level or inside EXPLAIN+ parent := ctx.GetParent()+ for parent != nil {+ switch parent.(type) {+ case *parser.ExplainstmtContext:+ // UPDATE inside EXPLAIN - mark as invalid+ l.hasInvalidStmt = true+ return+ case *parser.RootContext, *parser.StmtblockContext, *parser.StmtmultiContext, *parser.StmtContext, *parser.ExplainablestmtContext:+ // Continue walking up - ExplainablestmtContext is the wrapper for statements in EXPLAIN+ parent = parent.GetParent()+ default:+ // Hit some other context - this is nested+ return+ }+ }+ // Reached root - this is a top-level UPDATE+ l.hasInvalidStmt = true+}++// EnterDeletestmt detects DELETE statements (not allowed at top level)+func (l *queryValidatorListener) EnterDeletestmt(ctx *parser.DeletestmtContext) {+ // Check if this is at top level or inside EXPLAIN+ parent := ctx.GetParent()+ for parent != nil {+ switch parent.(type) {+ case *parser.ExplainstmtContext:+ // DELETE inside EXPLAIN - mark as invalid+ l.hasInvalidStmt = true+ return+ case *parser.RootContext, *parser.StmtblockContext, *parser.StmtmultiContext, *parser.StmtContext, *parser.ExplainablestmtContext:+ // Continue walking up - ExplainablestmtContext is the wrapper for statements in EXPLAIN+ parent = parent.GetParent()+ default:+ // Hit some other context - this is nested+ return+ }+ }+ // Reached root - this is a top-level DELETE+ l.hasInvalidStmt = true+}++// isExplainAnalyze checks if an EXPLAIN statement has ANALYZE option+func (*queryValidatorListener) isExplainAnalyze(ctx *parser.ExplainstmtContext) bool {+ // Check for ANALYZE keyword in explain options+ if ctx.Explain_option_list() != nil {+ for _, optCtx := range ctx.Explain_option_list().AllExplain_option_elem() {+ if optCtx.Explain_option_name() != nil {+ optName := optCtx.Explain_option_name().GetText()+ if optName == "ANALYZE" || optName == "analyze" {+ return true+ }+ }+ }+ }++ // Also check for old-style EXPLAIN ANALYZE (without parentheses)+ // Look for ANALYZE keyword directly after EXPLAIN+ for _, child := range ctx.GetChildren() {+ if termNode, ok := child.(antlr.TerminalNode); ok {+ if termNode.GetText() == "ANALYZE" || termNode.GetText() == "analyze" {+ return true+ }+ }+ }++ return false+}++// isTopLevelStmt checks if the statement is at the top level (not nested in CTE, subquery, etc.)+func isTopLevelStmt(ctx antlr.ParserRuleContext) bool {+ parent := ctx.GetParent()+ for parent != nil {+ switch parent.(type) {+ case *parser.RootContext:+ return true+ case *parser.StmtblockContext, *parser.StmtmultiContext, *parser.StmtContext:+ // Continue walking up+ parent = parent.GetParent()+ default:+ // If we hit any other context type, it's nested+ return false+ }+ }+ return true+}++// dmlDetectorListener detects INSERT/UPDATE/DELETE statements anywhere in the tree+type dmlDetectorListener struct {+ *parser.BasePostgreSQLParserListener++ hasDML bool+}++// EnterInsertstmt detects INSERT+func (d *dmlDetectorListener) EnterInsertstmt(_ *parser.InsertstmtContext) {+ d.hasDML = true+}++// EnterUpdatestmt detects UPDATE+func (d *dmlDetectorListener) EnterUpdatestmt(_ *parser.UpdatestmtContext) {+ d.hasDML = true+}++// EnterDeletestmt detects DELETE+func (d *dmlDetectorListener) EnterDeletestmt(_ *parser.DeletestmtContext) {+ d.hasDML = true+}diff --git a/backend/plugin/parser/pg/query_test.go b/backend/plugin/parser/pg/query_test.goindex fd55222c62..3a9df19dc9 100644--- a/backend/plugin/parser/pg/query_test.go+++ b/backend/plugin/parser/pg/query_test.go@@ -99,7 +99,7 @@ func TestValidateSQLForEditor(t *testing.T) {}for _, test := range tests {- gotValid, gotAllQuery, err := validateQuery(test.sql)+ gotValid, gotAllQuery, err := validateQueryANTLR(test.sql)require.NoError(t, err)require.Equal(t, test.valid, gotValid, test.sql)require.Equal(t, test.allQuery, gotAllQuery, test.sql)