Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(starrocks): replace ANTLR with StarRocks omni parser for LIMIT rewriting (#20655)
rebelice7881924307f7
diff --git a/backend/plugin/db/mysql/query_test.go b/backend/plugin/db/mysql/query_test.goindex b36998d30f..e86281819a 100644--- a/backend/plugin/db/mysql/query_test.go+++ b/backend/plugin/db/mysql/query_test.go@@ -195,6 +195,21 @@ func TestGetStatementWithResultLimit(t *testing.T) {count: 10,want: "SELECT * INTO OUTFILE '/tmp/a' FROM t LIMIT 10;",},+ {+ stmt: "SELECT * FROM t -- note",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 -- note",+ },+ {+ stmt: "SELECT * FROM t /* block comment */;",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",+ },+ {+ stmt: "SELECT * FROM t # mysql comment",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 # mysql comment",+ },}for _, tc := range testCases {diff --git a/backend/plugin/db/starrocks/antlr_dependency_test.go b/backend/plugin/db/starrocks/antlr_dependency_test.gonew file mode 100644index 0000000000..b5c1b41753--- /dev/null+++ b/backend/plugin/db/starrocks/antlr_dependency_test.go@@ -0,0 +1,23 @@+package starrocks++import (+ "os"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestStarRocksQueryDoesNotUseANTLROrMySQLParser(t *testing.T) {+ content, err := os.ReadFile("query.go")+ require.NoError(t, err)+ source := string(content)++ require.NotContains(t, source, "github.com/antlr4-go/antlr/v4")+ require.NotContains(t, source, "github.com/bytebase/parser/mysql")+ require.NotContains(t, source, "ParseMySQL(")+ require.NotContains(t, source, "TokenStreamRewriter")+ require.NotContains(t, source, "BaseMySQLParserListener")+ require.NotContains(t, source, "omni/mysql/ast")+ require.NotContains(t, source, "omni/mysql/parser")+ require.NotContains(t, source, "plugin/parser/mysql")+}diff --git a/backend/plugin/db/starrocks/query.go b/backend/plugin/db/starrocks/query.goindex 707dee7e86..fb0de9dd42 100644--- a/backend/plugin/db/starrocks/query.go+++ b/backend/plugin/db/starrocks/query.go@@ -7,15 +7,13 @@ import ("strconv""strings"- "github.com/antlr4-go/antlr/v4"+ "github.com/bytebase/omni/starrocks/ast"+ "github.com/bytebase/omni/starrocks/parser""github.com/pkg/errors"- "github.com/bytebase/parser/mysql"-"github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util"- mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {@@ -87,84 +85,208 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.}func getStatementWithResultLimit(statement string, limit int) string {- // SHOW statements don't need LIMIT clause wrapping as they typically return small result sets- // and wrapping them in SELECT * FROM (...) causes syntax errorstrimmedStatement := strings.TrimSpace(statement)if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {return statement}- stmt, err := getStatementWithResultLimitForMySQL(statement, limit)+ stmt, err := getStatementWithResultLimitInline(statement, limit)if err != nil {slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- // MySQL 5.7 doesn't support WITH clause.return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)}return stmt}-// singleStatement must be a selectStatement for mysql.-func getStatementWithResultLimitForMySQL(statement string, limitCount int) (string, error) {- list, err := mysqlparser.ParseMySQL(statement)- if err != nil {- return "", err+func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")}- listener := &mysqlRewriter{- limitCount: limitCount,- outerMostQuery: true,+ file, errs := parser.Parse(statement)+ if len(errs) > 0 {+ return "", errors.New(errs[0].Error())+ }+ if file == nil || len(file.Stmts) != 1 {+ stmtCount := 0+ if file != nil {+ stmtCount = len(file.Stmts)+ }+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)}- for _, stmt := range list {- listener.rewriter = *antlr.NewTokenStreamRewriter(stmt.Tokens)- antlr.ParseTreeWalkerDefault.Walk(listener, stmt.Tree)- if listener.err != nil {- return "", errors.Wrapf(listener.err, "statement: %s", statement)+ switch stmt := file.Stmts[0].(type) {+ case *ast.SelectStmt:+ return rewriteSelectLimit(statement, stmt, limitCount)+ case *ast.SetOpStmt:+ return rewriteSetOpLimit(statement, stmt, limitCount)+ case *ast.ParenSelect:+ switch inner := stmt.Sel.(type) {+ case *ast.SelectStmt:+ return rewriteSelectLimit(statement, inner, limitCount)+ case *ast.SetOpStmt:+ return rewriteSetOpLimit(statement, inner, limitCount)}+ return statement, nil+ default:+ return statement, nil}- return listener.rewriter.GetTextDefault(), nil}-type mysqlRewriter struct {- *mysql.BaseMySQLParserListener-- rewriter antlr.TokenStreamRewriter- err error- outerMostQuery bool- limitCount int+func hasUnparsedTail(sql string, locEnd int) bool {+ pos := locEnd+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ for pos < len(sql) && sql[pos] == ';' {+ pos+++ }+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if pos >= len(sql) {+ return false+ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return false+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return false+ }+ pos += end + 2+ continue+ }+ return true+ }+ return false}-func (r *mysqlRewriter) EnterQueryExpression(ctx *mysql.QueryExpressionContext) {- if !r.outerMostQuery {- return+func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {+ if stmt.Limit != nil {+ limitLoc := literalLoc(stmt.Limit)+ if limitLoc.Start < 0 {+ return "", errors.New("cannot rewrite non-constant LIMIT expression")+ }++ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])+ if existingCount > 0 && existingCount <= limitCount {+ return sql, nil+ }+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil+ }++ existingLimit := extractLimitValue(stmt.Limit)+ if existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil+ }+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil+ }++ if stmt.Into != nil {+ return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil+ }+ if hasUnparsedTail(sql, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")}- r.outerMostQuery = false- limitClause := ctx.LimitClause()- if limitClause != nil && limitClause.LimitOptions() != nil && len(limitClause.LimitOptions().AllLimitOption()) > 0 {- userLimitOption := limitClause.LimitOptions().LimitOption(0)- if limitClause.LimitOptions().COMMA_SYMBOL() != nil {- userLimitOption = limitClause.LimitOptions().LimitOption(1)+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil+}++func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {+ if stmt.Limit != nil {+ limitLoc := literalLoc(stmt.Limit)+ if limitLoc.Start < 0 {+ return "", errors.New("cannot rewrite non-constant LIMIT expression")+ }++ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])+ if existingCount > 0 && existingCount <= limitCount {+ return sql, nil+ }+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil}- userLimitText := userLimitOption.GetText()- limit, _ := strconv.Atoi(userLimitText)- if limit == 0 || r.limitCount < limit {- limit = r.limitCount+ existingLimit := extractLimitValue(stmt.Limit)+ if existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil}- r.rewriter.ReplaceDefault(userLimitOption.GetStart().GetTokenIndex(), userLimitOption.GetStop().GetTokenIndex(), fmt.Sprintf("%d", limit))- return+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil+ }+ if hasUnparsedTail(sql, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")+ }+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil+}++func extractLimitValue(node ast.Node) int {+ lit, ok := node.(*ast.Literal)+ if !ok || lit.Kind != ast.LitInt {+ return -1+ }+ val, err := strconv.Atoi(lit.Value)+ if err != nil {+ return -1+ }+ return val+}++func literalLoc(node ast.Node) ast.Loc {+ lit, ok := node.(*ast.Literal)+ if !ok {+ return ast.Loc{Start: -1, End: -1}+ }+ return lit.Loc+}++func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {+ pos := limitLoc.End+ pos = skipWhitespaceAndComments(sql, pos)+ if pos >= len(sql) || sql[pos] != ',' {+ return 0, 0, false+ }+ pos+++ pos = skipWhitespaceAndComments(sql, pos)+ countStart = pos+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {+ pos++}+ if pos == countStart {+ return 0, 0, false+ }+ return countStart, pos, true+}- if ctx.OrderClause() != nil {- r.rewriter.InsertAfterDefault(ctx.OrderClause().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))- } else {- switch {- case ctx.QueryExpressionBody() != nil:- r.rewriter.InsertAfterDefault(ctx.QueryExpressionBody().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))- case ctx.QueryExpressionParens() != nil:- r.rewriter.InsertAfterDefault(ctx.QueryExpressionParens().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))- default:- // No query expression body or parens found, skip adding LIMIT+func skipWhitespaceAndComments(sql string, pos int) int {+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return len(sql)+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return len(sql)+ }+ pos += end + 2+ continue}+ return pos}+ return pos}diff --git a/backend/plugin/db/starrocks/query_test.go b/backend/plugin/db/starrocks/query_test.goindex f6a387f058..22653f4b99 100644--- a/backend/plugin/db/starrocks/query_test.go+++ b/backend/plugin/db/starrocks/query_test.go@@ -69,16 +69,15 @@ func TestGetStatementWithResultLimit(t *testing.T) {count: 10,want: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count LIMIT 10;",},- // EXCEPT need mysql >= 8.0.31- // {- // stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",- // count: 20,- // want: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers LIMIT 10;",- // },+ {+ stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",+ count: 10,+ want: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers LIMIT 10;",+ },{stmt: "SELECT col1, col2 FROM table1 PROCEDURE ANALYSE(10, 2000);",count: 10,- want: "SELECT col1, col2 FROM table1 LIMIT 10 PROCEDURE ANALYSE(10, 2000);",+ want: "SELECT * FROM (SELECT col1, col2 FROM table1 PROCEDURE ANALYSE(10, 2000)) result LIMIT 10;",},{stmt: "SELECT col1, col2 FROM table1 ORDER BY col1;",@@ -95,6 +94,16 @@ func TestGetStatementWithResultLimit(t *testing.T) {count: 10,want: "SELECT * FROM t LIMIT 123,10;",},+ {+ stmt: "SELECT * FROM t LIMIT 0,\n1000000;",+ count: 10,+ want: "SELECT * FROM t LIMIT 0,\n10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 0 /* offset comment */, 1000000;",+ count: 10,+ want: "SELECT * FROM t LIMIT 0 /* offset comment */, 10;",+ },// SHOW statements should not be wrapped{stmt: "SHOW DATA",@@ -126,6 +135,94 @@ func TestGetStatementWithResultLimit(t *testing.T) {count: 10,want: " SHOW DATA ",},+ {+ stmt: `SELECT * FROM orders INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "s3.amazonaws.com");`,+ count: 10,+ want: `SELECT * FROM orders LIMIT 10 INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "s3.amazonaws.com");`,+ },+ {+ stmt: "SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;",+ count: 10,+ want: "SELECT * FROM (SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age) result LIMIT 10;",+ },+ {+ stmt: "SELECT * FROM t -- note",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 -- note",+ },+ {+ stmt: "SELECT * FROM t /* block comment */;",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",+ },+ {+ stmt: "SELECT * FROM t # hash comment",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 # hash comment",+ },+ {+ stmt: "SELECT * FROM t;\n",+ count: 10,+ want: "SELECT * FROM t LIMIT 10;\n",+ },+ {+ stmt: "SELECT * FROM t; \n",+ count: 10,+ want: "SELECT * FROM t LIMIT 10; \n",+ },+ // UNION with existing outer LIMIT — rewrite if larger+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100;",+ count: 10,+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 10;",+ },+ // UNION with existing outer LIMIT — keep if smaller+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 5;",+ count: 10,+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 5;",+ },+ // INTERSECT with no LIMIT+ {+ stmt: "SELECT a FROM t1 INTERSECT SELECT b FROM t2;",+ count: 10,+ want: "SELECT a FROM t1 INTERSECT SELECT b FROM t2 LIMIT 10;",+ },+ // Parenthesized query with branch-local LIMIT — outer LIMIT added+ {+ stmt: "SELECT a FROM t1 UNION (SELECT b FROM t2 LIMIT 1);",+ count: 10,+ want: "SELECT a FROM t1 UNION (SELECT b FROM t2 LIMIT 1) LIMIT 10;",+ },+ // INTO OUTFILE with existing LIMIT+ {+ stmt: `SELECT * FROM orders LIMIT 100 INTO OUTFILE "s3://bucket/export/";`,+ count: 10,+ want: `SELECT * FROM orders LIMIT 10 INTO OUTFILE "s3://bucket/export/";`,+ },+ // Comma-style LIMIT on UNION+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,1000000;",+ count: 10,+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,10;",+ },+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0 /* offset comment */, 1000000;",+ count: 10,+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0 /* offset comment */, 10;",+ },+ // Comma-style LIMIT on UNION — keep if smaller+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,5;",+ count: 10,+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,5;",+ },+ // Comment followed by unparsed content should still trigger CTE fallback+ {+ stmt: "SELECT * FROM person /* c */ LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;",+ count: 10,+ want: "SELECT * FROM (SELECT * FROM person /* c */ LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age) result LIMIT 10;",+ },}for _, tc := range testCases {diff --git a/go.mod b/go.modindex b54b8306a8..38448e7b59 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f+ github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3github.com/caarlos0/env/v11 v11.4.1github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 96e35d24d9..ff239e8278 100644--- a/go.sum+++ b/go.sum@@ -250,8 +250,8 @@ github.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352 h1:3sC5pdvJ7bNAhRgithub.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352/go.mod h1:eyWrxE51HCbjGIChvty/mYaNcAroXshnIiR6SIqlxfY=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f h1:YzyfON4c8NJwddIvQlOtuqqONLjK+vZvLbJcZx4r9vA=-github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f/go.mod h1:Gp5RN3FM07f9/FFOTZCeu8duTAIMeuinkjjBCrs2Dw4=+github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3 h1:OzvEFEDDAuf1TALMF9cuFsISPCF0nchW+wCUv/6o8PQ=+github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3/go.mod h1:Gp5RN3FM07f9/FFOTZCeu8duTAIMeuinkjjBCrs2Dw4=github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640 h1:g4Jed1/Jv/DHBiQmEgN5ILOQDDBjFlG4N4Lp+3B4aYE=github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=