Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(starrocks): replace ANTLR with StarRocks omni parser for LIMIT rewriting (#20655)

rebelice7881924307f7
diff --git a/backend/plugin/db/mysql/query_test.go b/backend/plugin/db/mysql/query_test.go
index b36998d30f..e86281819a 100644
--- a/backend/plugin/db/mysql/query_test.go
+++ b/backend/plugin/db/mysql/query_test.go
@@ -195,6 +195,21 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "SELECT * INTO OUTFILE '/tmp/a' FROM t LIMIT 10;",
},
+ {
+ stmt: "SELECT * FROM t -- note",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 -- note",
+ },
+ {
+ stmt: "SELECT * FROM t /* block comment */;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",
+ },
+ {
+ stmt: "SELECT * FROM t # mysql comment",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 # mysql comment",
+ },
}
for _, tc := range testCases {
diff --git a/backend/plugin/db/starrocks/antlr_dependency_test.go b/backend/plugin/db/starrocks/antlr_dependency_test.go
new file mode 100644
index 0000000000..b5c1b41753
--- /dev/null
+++ b/backend/plugin/db/starrocks/antlr_dependency_test.go
@@ -0,0 +1,23 @@
+package starrocks
+
+import (
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestStarRocksQueryDoesNotUseANTLROrMySQLParser(t *testing.T) {
+ content, err := os.ReadFile("query.go")
+ require.NoError(t, err)
+ source := string(content)
+
+ require.NotContains(t, source, "github.com/antlr4-go/antlr/v4")
+ require.NotContains(t, source, "github.com/bytebase/parser/mysql")
+ require.NotContains(t, source, "ParseMySQL(")
+ require.NotContains(t, source, "TokenStreamRewriter")
+ require.NotContains(t, source, "BaseMySQLParserListener")
+ require.NotContains(t, source, "omni/mysql/ast")
+ require.NotContains(t, source, "omni/mysql/parser")
+ require.NotContains(t, source, "plugin/parser/mysql")
+}
diff --git a/backend/plugin/db/starrocks/query.go b/backend/plugin/db/starrocks/query.go
index 707dee7e86..fb0de9dd42 100644
--- a/backend/plugin/db/starrocks/query.go
+++ b/backend/plugin/db/starrocks/query.go
@@ -7,15 +7,13 @@ import (
"strconv"
"strings"
- "github.com/antlr4-go/antlr/v4"
+ "github.com/bytebase/omni/starrocks/ast"
+ "github.com/bytebase/omni/starrocks/parser"
"github.com/pkg/errors"
- "github.com/bytebase/parser/mysql"
-
"github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
- mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
)
func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {
@@ -87,84 +85,208 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.
}
func getStatementWithResultLimit(statement string, limit int) string {
- // SHOW statements don't need LIMIT clause wrapping as they typically return small result sets
- // and wrapping them in SELECT * FROM (...) causes syntax errors
trimmedStatement := strings.TrimSpace(statement)
if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {
return statement
}
- stmt, err := getStatementWithResultLimitForMySQL(statement, limit)
+ stmt, err := getStatementWithResultLimitInline(statement, limit)
if err != nil {
slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- // MySQL 5.7 doesn't support WITH clause.
return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)
}
return stmt
}
-// singleStatement must be a selectStatement for mysql.
-func getStatementWithResultLimitForMySQL(statement string, limitCount int) (string, error) {
- list, err := mysqlparser.ParseMySQL(statement)
- if err != nil {
- return "", err
+func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
}
- listener := &mysqlRewriter{
- limitCount: limitCount,
- outerMostQuery: true,
+ file, errs := parser.Parse(statement)
+ if len(errs) > 0 {
+ return "", errors.New(errs[0].Error())
+ }
+ if file == nil || len(file.Stmts) != 1 {
+ stmtCount := 0
+ if file != nil {
+ stmtCount = len(file.Stmts)
+ }
+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)
}
- for _, stmt := range list {
- listener.rewriter = *antlr.NewTokenStreamRewriter(stmt.Tokens)
- antlr.ParseTreeWalkerDefault.Walk(listener, stmt.Tree)
- if listener.err != nil {
- return "", errors.Wrapf(listener.err, "statement: %s", statement)
+ switch stmt := file.Stmts[0].(type) {
+ case *ast.SelectStmt:
+ return rewriteSelectLimit(statement, stmt, limitCount)
+ case *ast.SetOpStmt:
+ return rewriteSetOpLimit(statement, stmt, limitCount)
+ case *ast.ParenSelect:
+ switch inner := stmt.Sel.(type) {
+ case *ast.SelectStmt:
+ return rewriteSelectLimit(statement, inner, limitCount)
+ case *ast.SetOpStmt:
+ return rewriteSetOpLimit(statement, inner, limitCount)
}
+ return statement, nil
+ default:
+ return statement, nil
}
- return listener.rewriter.GetTextDefault(), nil
}
-type mysqlRewriter struct {
- *mysql.BaseMySQLParserListener
-
- rewriter antlr.TokenStreamRewriter
- err error
- outerMostQuery bool
- limitCount int
+func hasUnparsedTail(sql string, locEnd int) bool {
+ pos := locEnd
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ for pos < len(sql) && sql[pos] == ';' {
+ pos++
+ }
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if pos >= len(sql) {
+ return false
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return false
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return false
+ }
+ pos += end + 2
+ continue
+ }
+ return true
+ }
+ return false
}
-func (r *mysqlRewriter) EnterQueryExpression(ctx *mysql.QueryExpressionContext) {
- if !r.outerMostQuery {
- return
+func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil {
+ limitLoc := literalLoc(stmt.Limit)
+ if limitLoc.Start < 0 {
+ return "", errors.New("cannot rewrite non-constant LIMIT expression")
+ }
+
+ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {
+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])
+ if existingCount > 0 && existingCount <= limitCount {
+ return sql, nil
+ }
+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil
+ }
+
+ existingLimit := extractLimitValue(stmt.Limit)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil
+ }
+
+ if stmt.Into != nil {
+ return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil
+ }
+ if hasUnparsedTail(sql, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
}
- r.outerMostQuery = false
- limitClause := ctx.LimitClause()
- if limitClause != nil && limitClause.LimitOptions() != nil && len(limitClause.LimitOptions().AllLimitOption()) > 0 {
- userLimitOption := limitClause.LimitOptions().LimitOption(0)
- if limitClause.LimitOptions().COMMA_SYMBOL() != nil {
- userLimitOption = limitClause.LimitOptions().LimitOption(1)
+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
+}
+
+func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil {
+ limitLoc := literalLoc(stmt.Limit)
+ if limitLoc.Start < 0 {
+ return "", errors.New("cannot rewrite non-constant LIMIT expression")
+ }
+
+ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {
+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])
+ if existingCount > 0 && existingCount <= limitCount {
+ return sql, nil
+ }
+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil
}
- userLimitText := userLimitOption.GetText()
- limit, _ := strconv.Atoi(userLimitText)
- if limit == 0 || r.limitCount < limit {
- limit = r.limitCount
+ existingLimit := extractLimitValue(stmt.Limit)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
}
- r.rewriter.ReplaceDefault(userLimitOption.GetStart().GetTokenIndex(), userLimitOption.GetStop().GetTokenIndex(), fmt.Sprintf("%d", limit))
- return
+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil
+ }
+ if hasUnparsedTail(sql, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
+ }
+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
+}
+
+func extractLimitValue(node ast.Node) int {
+ lit, ok := node.(*ast.Literal)
+ if !ok || lit.Kind != ast.LitInt {
+ return -1
+ }
+ val, err := strconv.Atoi(lit.Value)
+ if err != nil {
+ return -1
+ }
+ return val
+}
+
+func literalLoc(node ast.Node) ast.Loc {
+ lit, ok := node.(*ast.Literal)
+ if !ok {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ return lit.Loc
+}
+
+func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {
+ pos := limitLoc.End
+ pos = skipWhitespaceAndComments(sql, pos)
+ if pos >= len(sql) || sql[pos] != ',' {
+ return 0, 0, false
+ }
+ pos++
+ pos = skipWhitespaceAndComments(sql, pos)
+ countStart = pos
+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {
+ pos++
}
+ if pos == countStart {
+ return 0, 0, false
+ }
+ return countStart, pos, true
+}
- if ctx.OrderClause() != nil {
- r.rewriter.InsertAfterDefault(ctx.OrderClause().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))
- } else {
- switch {
- case ctx.QueryExpressionBody() != nil:
- r.rewriter.InsertAfterDefault(ctx.QueryExpressionBody().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))
- case ctx.QueryExpressionParens() != nil:
- r.rewriter.InsertAfterDefault(ctx.QueryExpressionParens().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))
- default:
- // No query expression body or parens found, skip adding LIMIT
+func skipWhitespaceAndComments(sql string, pos int) int {
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return len(sql)
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return len(sql)
+ }
+ pos += end + 2
+ continue
}
+ return pos
}
+ return pos
}
diff --git a/backend/plugin/db/starrocks/query_test.go b/backend/plugin/db/starrocks/query_test.go
index f6a387f058..22653f4b99 100644
--- a/backend/plugin/db/starrocks/query_test.go
+++ b/backend/plugin/db/starrocks/query_test.go
@@ -69,16 +69,15 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count LIMIT 10;",
},
- // EXCEPT need mysql >= 8.0.31
- // {
- // stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",
- // count: 20,
- // want: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers LIMIT 10;",
- // },
+ {
+ stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",
+ count: 10,
+ want: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers LIMIT 10;",
+ },
{
stmt: "SELECT col1, col2 FROM table1 PROCEDURE ANALYSE(10, 2000);",
count: 10,
- want: "SELECT col1, col2 FROM table1 LIMIT 10 PROCEDURE ANALYSE(10, 2000);",
+ want: "SELECT * FROM (SELECT col1, col2 FROM table1 PROCEDURE ANALYSE(10, 2000)) result LIMIT 10;",
},
{
stmt: "SELECT col1, col2 FROM table1 ORDER BY col1;",
@@ -95,6 +94,16 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "SELECT * FROM t LIMIT 123,10;",
},
+ {
+ stmt: "SELECT * FROM t LIMIT 0,\n1000000;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0,\n10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 0 /* offset comment */, 1000000;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0 /* offset comment */, 10;",
+ },
// SHOW statements should not be wrapped
{
stmt: "SHOW DATA",
@@ -126,6 +135,94 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: " SHOW DATA ",
},
+ {
+ stmt: `SELECT * FROM orders INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "s3.amazonaws.com");`,
+ count: 10,
+ want: `SELECT * FROM orders LIMIT 10 INTO OUTFILE "s3://bucket/export/" FORMAT AS PARQUET PROPERTIES("s3.endpoint" = "s3.amazonaws.com");`,
+ },
+ {
+ stmt: "SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;",
+ count: 10,
+ want: "SELECT * FROM (SELECT * FROM person LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age) result LIMIT 10;",
+ },
+ {
+ stmt: "SELECT * FROM t -- note",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 -- note",
+ },
+ {
+ stmt: "SELECT * FROM t /* block comment */;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",
+ },
+ {
+ stmt: "SELECT * FROM t # hash comment",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 # hash comment",
+ },
+ {
+ stmt: "SELECT * FROM t;\n",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10;\n",
+ },
+ {
+ stmt: "SELECT * FROM t; \n",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10; \n",
+ },
+ // UNION with existing outer LIMIT — rewrite if larger
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100;",
+ count: 10,
+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 10;",
+ },
+ // UNION with existing outer LIMIT — keep if smaller
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 5;",
+ count: 10,
+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 5;",
+ },
+ // INTERSECT with no LIMIT
+ {
+ stmt: "SELECT a FROM t1 INTERSECT SELECT b FROM t2;",
+ count: 10,
+ want: "SELECT a FROM t1 INTERSECT SELECT b FROM t2 LIMIT 10;",
+ },
+ // Parenthesized query with branch-local LIMIT — outer LIMIT added
+ {
+ stmt: "SELECT a FROM t1 UNION (SELECT b FROM t2 LIMIT 1);",
+ count: 10,
+ want: "SELECT a FROM t1 UNION (SELECT b FROM t2 LIMIT 1) LIMIT 10;",
+ },
+ // INTO OUTFILE with existing LIMIT
+ {
+ stmt: `SELECT * FROM orders LIMIT 100 INTO OUTFILE "s3://bucket/export/";`,
+ count: 10,
+ want: `SELECT * FROM orders LIMIT 10 INTO OUTFILE "s3://bucket/export/";`,
+ },
+ // Comma-style LIMIT on UNION
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,1000000;",
+ count: 10,
+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,10;",
+ },
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0 /* offset comment */, 1000000;",
+ count: 10,
+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0 /* offset comment */, 10;",
+ },
+ // Comma-style LIMIT on UNION — keep if smaller
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,5;",
+ count: 10,
+ want: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 0,5;",
+ },
+ // Comment followed by unparsed content should still trigger CTE fallback
+ {
+ stmt: "SELECT * FROM person /* c */ LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age;",
+ count: 10,
+ want: "SELECT * FROM (SELECT * FROM person /* c */ LATERAL VIEW EXPLODE(ARRAY(30, 60)) tableName AS c_age) result LIMIT 10;",
+ },
}
for _, tc := range testCases {
diff --git a/go.mod b/go.mod
index b54b8306a8..38448e7b59 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
- github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f
+ github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3
github.com/caarlos0/env/v11 v11.4.1
github.com/cenkalti/backoff/v5 v5.0.3
github.com/cockroachdb/cockroachdb-parser v0.25.2
diff --git a/go.sum b/go.sum
index 96e35d24d9..ff239e8278 100644
--- a/go.sum
+++ b/go.sum
@@ -250,8 +250,8 @@ github.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352 h1:3sC5pdvJ7bNAhR
github.com/bytebase/gomongo v0.0.0-20260510192859-8526a440b352/go.mod h1:eyWrxE51HCbjGIChvty/mYaNcAroXshnIiR6SIqlxfY=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
-github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f h1:YzyfON4c8NJwddIvQlOtuqqONLjK+vZvLbJcZx4r9vA=
-github.com/bytebase/omni v0.0.0-20260624184925-84d0fc49432f/go.mod h1:Gp5RN3FM07f9/FFOTZCeu8duTAIMeuinkjjBCrs2Dw4=
+github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3 h1:OzvEFEDDAuf1TALMF9cuFsISPCF0nchW+wCUv/6o8PQ=
+github.com/bytebase/omni v0.0.0-20260625023543-cd8cfff6bed3/go.mod h1:Gp5RN3FM07f9/FFOTZCeu8duTAIMeuinkjjBCrs2Dw4=
github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640 h1:g4Jed1/Jv/DHBiQmEgN5ILOQDDBjFlG4N4Lp+3B4aYE=
github.com/bytebase/parser v0.0.0-20260417075056-57b6ef7a2640/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=