Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(mysql): migrate query limit rewriter to omni (#20210)

rebelicea8df43ef97a9
diff --git a/backend/plugin/db/mysql/query.go b/backend/plugin/db/mysql/query.go
index 1ece484868..89b58c89b0 100644
--- a/backend/plugin/db/mysql/query.go
+++ b/backend/plugin/db/mysql/query.go
@@ -4,14 +4,18 @@ import (
"database/sql"
"fmt"
"log/slog"
+ "reflect"
"strconv"
+ "strings"
"time"
"github.com/antlr4-go/antlr/v4"
+ "github.com/bytebase/omni/mysql/ast"
+ omnimysqlparser "github.com/bytebase/omni/mysql/parser"
"github.com/pkg/errors"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/bytebase/parser/mysql"
+ antlrmysql "github.com/bytebase/parser/mysql"
"github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
@@ -120,10 +124,37 @@ func getStatementWithResultLimit(statement string, limit int) string {
}
func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ list, err := mysqlparser.ParseMySQLOmni(statement)
+ if err != nil {
+ if stmt, legacyErr := getStatementWithResultLimitInlineLegacy(statement, limitCount); legacyErr == nil {
+ return stmt, nil
+ }
+ return "", err
+ }
+ if len(list.Items) != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", len(list.Items))
+ }
+
+ stmt, ok := list.Items[0].(*ast.SelectStmt)
+ if !ok {
+ return statement, nil
+ }
+
+ return rewriteMySQLSelectLimit(statement, stmt, limitCount)
+}
+
+func getStatementWithResultLimitInlineLegacy(statement string, limitCount int) (string, error) {
list, err := mysqlparser.ParseMySQL(statement)
if err != nil {
return "", err
}
+ if len(list) != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", len(list))
+ }
listener := &mysqlRewriter{
limitCount: limitCount,
@@ -141,7 +172,7 @@ func getStatementWithResultLimitInline(statement string, limitCount int) (string
}
type mysqlRewriter struct {
- *mysql.BaseMySQLParserListener
+ *antlrmysql.BaseMySQLParserListener
rewriter antlr.TokenStreamRewriter
err error
@@ -149,7 +180,7 @@ type mysqlRewriter struct {
limitCount int
}
-func (r *mysqlRewriter) EnterQueryExpression(ctx *mysql.QueryExpressionContext) {
+func (r *mysqlRewriter) EnterQueryExpression(ctx *antlrmysql.QueryExpressionContext) {
if !r.outerMostQuery {
return
}
@@ -179,7 +210,136 @@ func (r *mysqlRewriter) EnterQueryExpression(ctx *mysql.QueryExpressionContext)
case ctx.QueryExpressionParens() != nil:
r.rewriter.InsertAfterDefault(ctx.QueryExpressionParens().GetStop().GetTokenIndex(), fmt.Sprintf(" LIMIT %d", r.limitCount))
default:
- // No action needed for other query expression types
+ // No action needed for other query expression types.
+ }
+ }
+}
+
+func rewriteMySQLSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil && stmt.Limit.Count != nil {
+ existingLimit := extractMySQLLimit(stmt.Limit.Count)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+ loc := nodeLocOf(stmt.Limit.Count)
+ loc = trimMySQLLocSpace(sql, loc)
+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
+ if existingLimit < 0 && stmt.Into == nil {
+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
+ }
+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
+ }
+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
+ }
+
+ insertPos, beforeClause := findMySQLLimitInsertPosition(sql, stmt)
+ if insertPos < 0 || insertPos > len(sql) {
+ return "", errors.Errorf("invalid LIMIT insert position %d", insertPos)
+ }
+ if beforeClause {
+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
+ }
+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
+}
+
+func findMySQLLimitInsertPosition(sql string, stmt *ast.SelectStmt) (int, bool) {
+ if stmt.Into != nil && stmt.Into.Loc.Start > 0 {
+ intoStart := findMySQLKeywordBefore(sql, stmt.Into.Loc.Start, "INTO")
+ if intoStart >= maxMySQLLocEndBeforeTailClauses(stmt) {
+ return intoStart, true
+ }
+ }
+ if stmt.ForUpdate != nil && stmt.ForUpdate.Loc.Start > 0 {
+ return stmt.ForUpdate.Loc.Start, true
+ }
+ if stmt.Loc.End > 0 {
+ return stmt.Loc.End, false
+ }
+ return -1, false
+}
+
+func extractMySQLLimit(node ast.Node) int {
+ limit, ok := node.(*ast.IntLit)
+ if !ok {
+ return -1
+ }
+ return int(limit.Value)
+}
+
+func nodeLocOf(node ast.Node) ast.Loc {
+ return mysqlNodeLoc(node)
+}
+
+func trimMySQLLocSpace(sql string, loc ast.Loc) ast.Loc {
+ for loc.Start < loc.End && loc.Start < len(sql) && (sql[loc.Start] == ' ' || sql[loc.Start] == '\t' || sql[loc.Start] == '\n' || sql[loc.Start] == '\r') {
+ loc.Start++
+ }
+ for loc.End > loc.Start && loc.End <= len(sql) && (sql[loc.End-1] == ' ' || sql[loc.End-1] == '\t' || sql[loc.End-1] == '\n' || sql[loc.End-1] == '\r') {
+ loc.End--
+ }
+ return loc
+}
+
+func maxMySQLLocEndBeforeTailClauses(node ast.Node) int {
+ maxEnd := -1
+ ast.Inspect(node, func(n ast.Node) bool {
+ if n == nil {
+ return false
+ }
+ if n == node {
+ return true
+ }
+ if _, ok := n.(*ast.IntoClause); ok {
+ return false
+ }
+ if _, ok := n.(*ast.ForUpdate); ok {
+ return false
+ }
+ if loc := mysqlNodeLoc(n); loc.End > maxEnd {
+ maxEnd = loc.End
+ }
+ return true
+ })
+ return maxEnd
+}
+
+func mysqlNodeLoc(node ast.Node) ast.Loc {
+ if node == nil {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ value := reflect.ValueOf(node)
+ if value.Kind() != reflect.Pointer || value.IsNil() {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ elem := value.Elem()
+ if elem.Kind() != reflect.Struct {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ field := elem.FieldByName("Loc")
+ if !field.IsValid() || !field.CanInterface() {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ loc, ok := field.Interface().(ast.Loc)
+ if !ok {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ return loc
+}
+
+func findMySQLKeywordBefore(sql string, offset int, keyword string) int {
+ if offset > len(sql) {
+ offset = len(sql)
+ }
+ if len(keyword) == 0 {
+ return offset
+ }
+ keyword = strings.ToUpper(keyword)
+ tokens := omnimysqlparser.Tokenize(sql[:offset])
+ for i := len(tokens) - 1; i >= 0; i-- {
+ token := tokens[i]
+ if token.End <= offset && omnimysqlparser.TokenName(token.Type) == keyword {
+ return token.Loc
}
}
+ return offset
}
diff --git a/backend/plugin/db/mysql/query_test.go b/backend/plugin/db/mysql/query_test.go
index 777e4290af..77952cd13a 100644
--- a/backend/plugin/db/mysql/query_test.go
+++ b/backend/plugin/db/mysql/query_test.go
@@ -37,6 +37,11 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "SELECT * FROM t LIMIT 5;",
},
+ {
+ stmt: "SELECT * FROM t LIMIT 0;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0;",
+ },
{
stmt: "SELECT * FROM t LIMIT 123;",
count: 10,
@@ -95,6 +100,46 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "SELECT col1, col2 FROM table1 ORDER BY col1 LIMIT 10;",
},
+ {
+ stmt: "SELECT * FROM t FOR UPDATE;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 FOR UPDATE;",
+ },
+ {
+ stmt: "SELECT * FROM t LOCK IN SHARE MODE;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 LOCK IN SHARE MODE;",
+ },
+ {
+ stmt: "SELECT * FROM t INTO OUTFILE '/tmp/a';",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 INTO OUTFILE '/tmp/a';",
+ },
+ {
+ stmt: "SELECT * FROM t /* into */ INTO OUTFILE '/tmp/a';",
+ count: 10,
+ want: "SELECT * FROM t /* into */ LIMIT 10 INTO OUTFILE '/tmp/a';",
+ },
+ {
+ stmt: "SELECT * FROM t INTO /* into */ OUTFILE '/tmp/a';",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 INTO /* into */ OUTFILE '/tmp/a';",
+ },
+ {
+ stmt: "SELECT c INTO @v FROM t LIMIT row_count;",
+ count: 10,
+ want: "SELECT c INTO @v FROM t LIMIT 10;",
+ },
+ {
+ stmt: "SELECT c FROM t INTO @v FOR UPDATE;",
+ count: 10,
+ want: "SELECT c FROM t LIMIT 10 INTO @v FOR UPDATE;",
+ },
+ {
+ stmt: "SELECT * INTO OUTFILE '/tmp/a' FROM t;",
+ count: 10,
+ want: "SELECT * INTO OUTFILE '/tmp/a' FROM t LIMIT 10;",
+ },
}
for _, tc := range testCases {
@@ -102,3 +147,80 @@ func TestGetStatementWithResultLimit(t *testing.T) {
require.Equal(t, tc.want, got, tc.stmt)
}
}
+
+func TestGetStatementWithResultLimitFallback(t *testing.T) {
+ testCases := []struct {
+ stmt string
+ count int
+ want string
+ }{
+ {
+ stmt: "SELECT * FROM t LIMIT 1 + 2",
+ count: 5,
+ want: "SELECT * FROM (SELECT * FROM t LIMIT 1 + 2) result LIMIT 5;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT @row_count",
+ count: 5,
+ want: "SELECT * FROM (SELECT * FROM t LIMIT @row_count) result LIMIT 5;",
+ },
+ }
+
+ for _, tc := range testCases {
+ got := getStatementWithResultLimit(tc.stmt, tc.count)
+ require.Equal(t, tc.want, got, tc.stmt)
+ }
+}
+
+func TestGetStatementWithResultLimitInline(t *testing.T) {
+ testCases := []struct {
+ stmt string
+ count int
+ wantErr bool
+ want string
+ }{
+ {
+ stmt: "SELECT * FROM t LIMIT 1 + 2",
+ count: 5,
+ wantErr: true,
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT @row_count",
+ count: 5,
+ wantErr: true,
+ },
+ {
+ stmt: "SELECT * FROM a; SELECT * FROM b;",
+ count: 5,
+ wantErr: true,
+ },
+ {
+ stmt: "SELECT col1 FROM table1 PROCEDURE ANALYSE(10, 2000); SELECT * FROM b;",
+ count: 5,
+ wantErr: true,
+ },
+ {
+ stmt: "UPDATE t SET a = 1",
+ count: 5,
+ want: "UPDATE t SET a = 1",
+ },
+ }
+
+ for _, tc := range testCases {
+ got, err := getStatementWithResultLimitInline(tc.stmt, tc.count)
+ if tc.wantErr {
+ require.Error(t, err, tc.stmt)
+ continue
+ }
+ require.NoError(t, err, tc.stmt)
+ require.Equal(t, tc.want, got, tc.stmt)
+ }
+}
+
+func TestGetStatementWithResultLimitUnicodeBeforeInto(t *testing.T) {
+ stmt := "SELECT 'ı' FROM t INTO OUTFILE '/tmp/a';"
+ want := "SELECT 'ı' FROM t LIMIT 10 INTO OUTFILE '/tmp/a';"
+
+ got := getStatementWithResultLimit(stmt, 10)
+ require.Equal(t, want, got)
+}