Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser): eliminate internal double-splitting in MySQL and PostgreSQL parsers (#18939)
h3n4l0a858ba6febc
diff --git a/backend/plugin/parser/mysql/mysql.go b/backend/plugin/parser/mysql/mysql.goindex 1e5d06476c..483de01432 100644--- a/backend/plugin/parser/mysql/mysql.go+++ b/backend/plugin/parser/mysql/mysql.go@@ -1,7 +1,6 @@package mysqlimport (- "log/slog""regexp""strings"@@ -27,14 +26,14 @@ func init() {// parseMySQLStatements is the ParseStatementsFunc for MySQL, MariaDB, and OceanBase.// Returns []ParsedStatement with both text and AST populated.func parseMySQLStatements(statement string) ([]base.ParsedStatement, error) {- // First split to get Statement with text and positions+ // Split once to get Statement with text and positionsstmts, err := SplitSQL(statement)if err != nil {return nil, err}- // Then parse to get ASTs- parseResults, err := ParseMySQL(statement)+ // Parse using the pre-split statements to avoid double-splitting+ parseResults, err := parseMySQLStatementsInternal(stmts)if err != nil {return nil, err}@@ -62,12 +61,45 @@ func ParseMySQL(statement string) ([]*base.ANTLRAST, error) {if err != nil {return nil, err}- list, err := parseInputStream(antlr.NewInputStream(statement), statement)- // HACK(p0ny): the callee may end up in an infinite loop, we print the statement here to help debug.- if err != nil && strings.Contains(err.Error(), "split SQL statement timed out") {- slog.Info("split SQL statement timed out", "statement", statement)+ stmts, err := SplitSQL(statement)+ if err != nil {+ return nil, err+ }+ return parseMySQLStatementsInternal(stmts)+}++// parseMySQLStatementsInternal parses pre-split statements without re-splitting.+// This is the internal implementation used by both ParseMySQL and parseMySQLStatements.+func parseMySQLStatementsInternal(stmts []base.Statement) ([]*base.ANTLRAST, error) {+ var result []*base.ANTLRAST++ if len(stmts) > 0 {+ // Add semicolon to the last statement if needed+ stmts[len(stmts)-1].Text = mysqlAddSemicolonIfNeeded(stmts[len(stmts)-1].Text)+ }++ for _, s := range stmts {+ if s.Empty {+ continue+ }++ tree, tokens, err := parseSingleStatement(s.BaseLine(), s.Text)+ if err != nil {+ return nil, err+ }++ if isEmptyStatement(tokens) {+ continue+ }++ result = append(result, &base.ANTLRAST{+ StartPosition: &storepb.Position{Line: int32(s.BaseLine()) + 1},+ Tree: tree,+ Tokens: tokens,+ })}- return list, err++ return result, nil}// DealWithDelimiter converts the delimiter statement to comment, also converts the following statement's delimiter to semicolon(`;`).@@ -177,43 +209,6 @@ func mysqlAddSemicolonIfNeeded(sql string) string {return sql}-func parseInputStream(input *antlr.InputStream, statement string) ([]*base.ANTLRAST, error) {- var result []*base.ANTLRAST- lexer := parser.NewMySQLLexer(input)- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)-- list, err := splitMySQLStatement(stream, statement)- if err != nil {- return nil, err- }-- if len(list) > 0 {- list[len(list)-1].Text = mysqlAddSemicolonIfNeeded(list[len(list)-1].Text)- }-- baseLine := 0- for _, s := range list {- tree, tokens, err := parseSingleStatement(baseLine, s.Text)- if err != nil {- return nil, err- }-- if isEmptyStatement(tokens) {- continue- }-- result = append(result, &base.ANTLRAST{- StartPosition: &storepb.Position{Line: int32(s.BaseLine()) + 1},- Tree: tree,- Tokens: tokens,- })- // s.End.Line is 1-based, but baseLine should be 0-based- baseLine = int(s.End.Line) - 1- }-- return result, nil-}-func isEmptyStatement(tokens *antlr.CommonTokenStream) bool {for _, token := range tokens.GetAllTokens() {if token.GetChannel() == antlr.TokenDefaultChannel && token.GetTokenType() != parser.MySQLParserSEMICOLON_SYMBOL && token.GetTokenType() != parser.MySQLParserEOF {diff --git a/backend/plugin/parser/pg/pg.go b/backend/plugin/parser/pg/pg.goindex 79fbd56c42..fbf6ba1129 100644--- a/backend/plugin/parser/pg/pg.go+++ b/backend/plugin/parser/pg/pg.go@@ -18,14 +18,14 @@ func init() {// parsePgStatements is the ParseStatementsFunc for PostgreSQL.// Returns []ParsedStatement with both text and AST populated.func parsePgStatements(statement string) ([]base.ParsedStatement, error) {- // First split to get Statement with text and positions+ // Split once to get Statement with text and positionsstmts, err := SplitSQL(statement)if err != nil {return nil, err}- // Then parse to get ASTs- parseResults, err := ParsePostgreSQL(statement)+ // Parse using the pre-split statements to avoid double-splitting+ parseResults, err := parsePostgreSQLStatements(stmts)if err != nil {return nil, err}@@ -54,7 +54,12 @@ func ParsePostgreSQL(sql string) ([]*base.ANTLRAST, error) {if err != nil {return nil, err}+ return parsePostgreSQLStatements(stmts)+}+// parsePostgreSQLStatements parses pre-split statements without re-splitting.+// This is the internal implementation used by both ParsePostgreSQL and parsePgStatements.+func parsePostgreSQLStatements(stmts []base.Statement) ([]*base.ANTLRAST, error) {var results []*base.ANTLRASTfor _, stmt := range stmts {if stmt.Empty {