Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(sql): build LIMIT statements in the parsers (#21486)

Danny Xu7af93f500c3c
diff --git a/backend/plugin/db/bigquery/bigquery.go b/backend/plugin/db/bigquery/bigquery.go
index 94f02c1a18..b38d6c57fa 100644
--- a/backend/plugin/db/bigquery/bigquery.go
+++ b/backend/plugin/db/bigquery/bigquery.go
@@ -22,6 +22,10 @@ import (
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+
+ // Register how this engine plans a limit, for base.StatementWithResultLimit below.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/bigquery"
)
var (
@@ -167,7 +171,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
queryResult, err := func() (*v1pb.QueryResult, error) {
if util.IsSelect(statement) {
if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_BIGQUERY, statement, queryContext.Limit, "")
}
q := d.client.Query(statement)
if queryContext.OperatorEmail != "" {
@@ -360,14 +364,6 @@ func encodeOperatorEmail(email string) string {
return string(values)
}
-func getStatementWithResultLimit(statement string, limit int) string {
- limitPart := ""
- if limit > 0 {
- limitPart = fmt.Sprintf(" LIMIT %d", limit)
- }
- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", util.TrimStatement(statement), limitPart)
-}
-
func convertValue(v bigquery.Value, fieldType bigquery.FieldType) *v1pb.RowValue {
if v == nil {
return &v1pb.RowValue{Kind: &v1pb.RowValue_NullValue{NullValue: structpb.NullValue_NULL_VALUE}}
diff --git a/backend/plugin/db/clickhouse/query.go b/backend/plugin/db/clickhouse/query.go
index 4cc3a8ab9c..d37520ff93 100644
--- a/backend/plugin/db/clickhouse/query.go
+++ b/backend/plugin/db/clickhouse/query.go
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"encoding/json"
- "fmt"
"log/slog"
"math/big"
"reflect"
@@ -51,7 +50,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_CLICKHOUSE, statement, queryContext.Limit, "")
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_CLICKHOUSE, statement)
if err != nil {
@@ -106,10 +105,6 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,
return results, nil
}
-func getStatementWithResultLimit(statement string, limit int) string {
- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)
-}
-
// translateAggregateFunctionError rewrites the clickhouse-go driver's opaque
// "unsupported column type \"AggregateFunction(...)\"" error into actionable
// guidance pointing the user at -Merge combinators or finalizeAggregation(),
diff --git a/backend/plugin/db/cockroachdb/cockroachdb.go b/backend/plugin/db/cockroachdb/cockroachdb.go
index 0ba61e6a73..1a59eab2a1 100644
--- a/backend/plugin/db/cockroachdb/cockroachdb.go
+++ b/backend/plugin/db/cockroachdb/cockroachdb.go
@@ -628,7 +628,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, statement, queryContext.Limit, "")
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_POSTGRES, statement)
@@ -703,10 +703,3 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,
return results, nil
}
-
-func getStatementWithResultLimit(stmt string, limit int) string {
- // To handle cases where there are comments in the query.
- // eg. select * from t1 -- this is comment;
- // Add two new line symbol here.
- return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", util.TrimStatement(stmt), limit)
-}
diff --git a/backend/plugin/db/mssql/mssql.go b/backend/plugin/db/mssql/mssql.go
index 72647a08fe..df34b28b68 100644
--- a/backend/plugin/db/mssql/mssql.go
+++ b/backend/plugin/db/mssql/mssql.go
@@ -517,7 +517,7 @@ func (*Driver) queryBatch(ctx context.Context, conn *sql.Conn, batch string, que
// Before sending the batch to server, we add the limit clause to the statement.
s := singleSQL.Text
if queryContext.Limit > 0 {
- s = getStatementWithResultLimit(s, queryContext.Limit)
+ s = base.StatementWithResultLimit(storepb.Engine_MSSQL, s, queryContext.Limit, "")
}
refinedSQLs = append(refinedSQLs, s)
if _, err := batchBuf.WriteString(s); err != nil {
diff --git a/backend/plugin/db/mssql/query.go b/backend/plugin/db/mssql/query.go
index b23074d404..185f663879 100644
--- a/backend/plugin/db/mssql/query.go
+++ b/backend/plugin/db/mssql/query.go
@@ -2,16 +2,12 @@ package mssql
import (
"database/sql"
- "fmt"
- "log/slog"
"time"
- omnimssql "github.com/bytebase/omni/mssql"
"google.golang.org/protobuf/types/known/timestamppb"
mssqldb "github.com/microsoft/go-mssqldb"
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
)
@@ -157,16 +153,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.
}
return util.NullRowValue
}
-
-func getStatementWithResultLimit(statement string, limit int) string {
- stmt, err := getStatementWithResultLimitInline(statement, limit)
- if err != nil {
- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- return fmt.Sprintf("WITH result AS (%s) SELECT TOP %d * FROM result;", util.TrimStatement(statement), limit)
- }
- return stmt
-}
-
-func getStatementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {
- return omnimssql.StatementWithResultLimit(singleStatement, limitCount)
-}
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index 27be4889e5..a8842045ce 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -534,7 +534,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_MYSQL, statement, queryContext.Limit, "")
}
sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)
diff --git a/backend/plugin/db/mysql/query.go b/backend/plugin/db/mysql/query.go
index 3efcd65371..d322219626 100644
--- a/backend/plugin/db/mysql/query.go
+++ b/backend/plugin/db/mysql/query.go
@@ -2,19 +2,9 @@ package mysql
import (
"database/sql"
- "fmt"
- "log/slog"
- "reflect"
- "strings"
- "github.com/bytebase/omni/mysql/ast"
- omnimysqlparser "github.com/bytebase/omni/mysql/parser"
- "github.com/pkg/errors"
-
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
- mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql"
)
func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {
@@ -86,232 +76,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.
}
return util.NullRowValue
}
-
-func getStatementWithResultLimit(statement string, limit int) string {
- stmt, err := getStatementWithResultLimitInline(statement, limit)
- if err != nil {
- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- // MySQL 5.7 doesn't support WITH clause.
- return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)
- }
- return stmt
-}
-
-func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {
- if strings.TrimSpace(statement) == "" {
- return "", errors.New("empty statement")
- }
-
- list, err := mysqlparser.ParseMySQL(statement)
- if err != nil {
- if stmt, procedureErr := rewriteMySQLSelectProcedureLimit(statement, limitCount); procedureErr == nil {
- return stmt, nil
- }
- return "", err
- }
- if len(list.Items) != 1 {
- return "", errors.Errorf("expected exactly one statement, got %d", len(list.Items))
- }
-
- stmt, ok := list.Items[0].(*ast.SelectStmt)
- if !ok {
- return statement, nil
- }
-
- return rewriteMySQLSelectLimit(statement, stmt, limitCount)
-}
-
-func rewriteMySQLSelectProcedureLimit(sql string, limitCount int) (string, error) {
- statements, err := mysqlparser.SplitSQL(sql)
- if err != nil {
- return "", err
- }
- statementCount := 0
- for _, statement := range statements {
- if !statement.Empty {
- statementCount++
- }
- }
- if statementCount != 1 {
- return "", errors.Errorf("expected exactly one statement, got %d", statementCount)
- }
-
- tokens := omnimysqlparser.Tokenize(sql)
- if len(tokens) == 0 || omnimysqlparser.TokenName(tokens[0].Type) != "SELECT" {
- return "", errors.New("not a SELECT PROCEDURE statement")
- }
- for i, token := range tokens[1:] {
- if omnimysqlparser.TokenName(token.Type) == "PROCEDURE" {
- procedureTokenIndex := i + 1
- if stmt, ok := rewriteMySQLLimitBeforeProcedure(sql, tokens[:procedureTokenIndex], limitCount); ok {
- return stmt, nil
- }
- return sql[:token.Loc] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[token.Loc:], nil
- }
- }
- return "", errors.New("SELECT PROCEDURE clause not found")
-}
-
-func rewriteMySQLLimitBeforeProcedure(sql string, tokens []omnimysqlparser.Token, limitCount int) (string, bool) {
- depth := 0
- for i := len(tokens) - 1; i >= 0; i-- {
- switch tokens[i].Str {
- case ")":
- depth++
- continue
- case "(":
- if depth > 0 {
- depth--
- }
- continue
- default:
- }
- if depth > 0 {
- continue
- }
- if omnimysqlparser.TokenName(tokens[i].Type) != "LIMIT" {
- continue
- }
- countTokenIndex := i + 1
- if i+3 < len(tokens) && tokens[i+2].Str == "," {
- countTokenIndex = i + 3
- }
- if countTokenIndex >= len(tokens) {
- return "", false
- }
- if tokens[countTokenIndex].Ival > 0 && int(tokens[countTokenIndex].Ival) <= limitCount {
- return sql, true
- }
- return sql[:tokens[countTokenIndex].Loc] + fmt.Sprintf("%d", limitCount) + sql[tokens[countTokenIndex].End:], true
- }
- return "", false
-}
-
-func rewriteMySQLSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
- if stmt.Limit != nil && stmt.Limit.Count != nil {
- existingLimit := extractMySQLLimit(stmt.Limit.Count)
- if existingLimit >= 0 && existingLimit <= limitCount {
- return sql, nil
- }
- loc := nodeLocOf(stmt.Limit.Count)
- loc = trimMySQLLocSpace(sql, loc)
- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
- if existingLimit < 0 && stmt.Into == nil {
- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
- }
- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
- }
- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
- }
-
- insertPos, beforeClause := findMySQLLimitInsertPosition(sql, stmt)
- if insertPos < 0 || insertPos > len(sql) {
- return "", errors.Errorf("invalid LIMIT insert position %d", insertPos)
- }
- if beforeClause {
- return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
- }
- return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
-}
-
-func findMySQLLimitInsertPosition(sql string, stmt *ast.SelectStmt) (int, bool) {
- if stmt.Into != nil && stmt.Into.Loc.Start > 0 {
- intoStart := findMySQLKeywordBefore(sql, stmt.Into.Loc.Start, "INTO")
- if intoStart >= maxMySQLLocEndBeforeTailClauses(stmt) {
- return intoStart, true
- }
- }
- if stmt.ForUpdate != nil && stmt.ForUpdate.Loc.Start > 0 {
- return stmt.ForUpdate.Loc.Start, true
- }
- if stmt.Loc.End > 0 {
- return stmt.Loc.End, false
- }
- return -1, false
-}
-
-func extractMySQLLimit(node ast.Node) int {
- limit, ok := node.(*ast.IntLit)
- if !ok {
- return -1
- }
- return int(limit.Value)
-}
-
-func nodeLocOf(node ast.Node) ast.Loc {
- return mysqlNodeLoc(node)
-}
-
-func trimMySQLLocSpace(sql string, loc ast.Loc) ast.Loc {
- for loc.Start < loc.End && loc.Start < len(sql) && (sql[loc.Start] == ' ' || sql[loc.Start] == '\t' || sql[loc.Start] == '\n' || sql[loc.Start] == '\r') {
- loc.Start++
- }
- for loc.End > loc.Start && loc.End <= len(sql) && (sql[loc.End-1] == ' ' || sql[loc.End-1] == '\t' || sql[loc.End-1] == '\n' || sql[loc.End-1] == '\r') {
- loc.End--
- }
- return loc
-}
-
-func maxMySQLLocEndBeforeTailClauses(node ast.Node) int {
- maxEnd := -1
- ast.Inspect(node, func(n ast.Node) bool {
- if n == nil {
- return false
- }
- if n == node {
- return true
- }
- if _, ok := n.(*ast.IntoClause); ok {
- return false
- }
- if _, ok := n.(*ast.ForUpdate); ok {
- return false
- }
- if loc := mysqlNodeLoc(n); loc.End > maxEnd {
- maxEnd = loc.End
- }
- return true
- })
- return maxEnd
-}
-
-func mysqlNodeLoc(node ast.Node) ast.Loc {
- if node == nil {
- return ast.Loc{Start: -1, End: -1}
- }
- value := reflect.ValueOf(node)
- if value.Kind() != reflect.Pointer || value.IsNil() {
- return ast.Loc{Start: -1, End: -1}
- }
- elem := value.Elem()
- if elem.Kind() != reflect.Struct {
- return ast.Loc{Start: -1, End: -1}
- }
- field := elem.FieldByName("Loc")
- if !field.IsValid() || !field.CanInterface() {
- return ast.Loc{Start: -1, End: -1}
- }
- loc, ok := field.Interface().(ast.Loc)
- if !ok {
- return ast.Loc{Start: -1, End: -1}
- }
- return loc
-}
-
-func findMySQLKeywordBefore(sql string, offset int, keyword string) int {
- if offset > len(sql) {
- offset = len(sql)
- }
- if len(keyword) == 0 {
- return offset
- }
- keyword = strings.ToUpper(keyword)
- tokens := omnimysqlparser.Tokenize(sql[:offset])
- for i := len(tokens) - 1; i >= 0; i-- {
- token := tokens[i]
- if token.End <= offset && omnimysqlparser.TokenName(token.Type) == keyword {
- return token.Loc
- }
- }
- return offset
-}
diff --git a/backend/plugin/db/oracle/oracle.go b/backend/plugin/db/oracle/oracle.go
index 1773298036..4ac7098d63 100644
--- a/backend/plugin/db/oracle/oracle.go
+++ b/backend/plugin/db/oracle/oracle.go
@@ -250,7 +250,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
if !queryContext.Explain && queryContext.Limit > 0 {
- statement = addResultLimit(statement, queryContext.Limit, d.connectionCtx.EngineVersion)
+ statement = base.StatementWithResultLimit(storepb.Engine_ORACLE, statement, queryContext.Limit, d.connectionCtx.EngineVersion)
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_ORACLE, statement)
diff --git a/backend/plugin/db/oracle/query.go b/backend/plugin/db/oracle/query.go
index bc9586df95..32ebe2be64 100644
--- a/backend/plugin/db/oracle/query.go
+++ b/backend/plugin/db/oracle/query.go
@@ -2,27 +2,14 @@ package oracle
import (
"database/sql"
- "fmt"
- "log/slog"
- "strconv"
- "strings"
"time"
- "unicode"
- oracleast "github.com/bytebase/omni/oracle/ast"
- oracleparser "github.com/bytebase/omni/oracle/parser"
- "github.com/pkg/errors"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
- plsqlparser "github.com/bytebase/bytebase/backend/plugin/parser/plsql"
- "github.com/bytebase/bytebase/backend/utils"
)
-const dbVersion12 = 12
-
// ========== Type Conversion Functions ==========
// makeValueByTypeName creates appropriate Go types for Oracle column types.
@@ -153,284 +140,3 @@ func convertTimestamp(typeName string, columnType *sql.ColumnType, t time.Time)
}
}
}
-
-// ========== Limit Handling Functions ==========
-
-// addResultLimit adds a limit clause to the statement based on Oracle version
-func addResultLimit(stmt string, limit int, engineVersion string) string {
- // Check if we should skip adding limit (e.g., for simple DUAL queries)
- if shouldSkipLimit(stmt) {
- return stmt
- }
-
- // Determine Oracle version
- if isOracle11gOrEarlier(engineVersion) {
- return addLimitFor11g(stmt, limit)
- }
- return addLimitFor12cAndLater(stmt, limit)
-}
-
-// shouldSkipLimit checks if the statement needs a limit clause
-func shouldSkipLimit(stmt string) bool {
- ok, err := skipAddLimit(stmt)
- return err == nil && ok
-}
-
-// isOracle11gOrEarlier checks if the Oracle version is 11g or earlier
-func isOracle11gOrEarlier(engineVersion string) bool {
- versionIdx := strings.Index(engineVersion, ".")
- if versionIdx < 0 {
- return true // Default to 11g behavior for invalid version
- }
- versionNumber, err := strconv.Atoi(engineVersion[:versionIdx])
- if err != nil {
- return true // Default to 11g behavior for parsing errors
- }
- return versionNumber < dbVersion12
-}
-
-// addLimitFor11g adds a ROWNUM-based limit for Oracle 11g and earlier versions.
-// Uses the legacy approach with subquery and ROWNUM.
-func addLimitFor11g(statement string, limitCount int) string {
- if !isSelectOrWithStatement(statement) {
- return statement
- }
- return fmt.Sprintf("SELECT * FROM (%s) WHERE ROWNUM <= %d", util.TrimStatement(statement), limitCount)
-}
-
-// isSelectOrWithStatement checks if the statement is a SELECT or WITH statement
-func isSelectOrWithStatement(statement string) bool {
- trimmedStatement := strings.ToLower(strings.TrimLeftFunc(statement, unicode.IsSpace))
- return strings.HasPrefix(trimmedStatement, "select") || strings.HasPrefix(trimmedStatement, "with")
-}
-
-// addLimitFor12cAndLater adds a FETCH NEXT clause for Oracle 12c and later versions.
-// Uses the modern SQL standard approach, falling back to 11g approach on error.
-func addLimitFor12cAndLater(statement string, limit int) string {
- if !isSelectOrWithStatement(statement) {
- return statement
- }
-
- stmt, err := addFetchNextClause(statement, limit)
- if err != nil {
- slog.Error("failed to add FETCH NEXT clause, falling back to ROWNUM",
- slog.String("statement", statement), log.BBError(err))
- return addLimitFor11g(statement, limit)
- }
- return stmt
-}
-
-// addFetchNextClause adds a FETCH NEXT clause to a SELECT statement using AST parsing.
-// This provides more precise placement of the limit clause compared to simple string wrapping.
-func addFetchNextClause(statement string, limitCount int) (string, error) {
- list, err := plsqlparser.ParsePLSQL(statement)
- if err != nil {
- return "", err
- }
- if list == nil || len(list.Items) == 0 {
- return "", errors.New("no parse results")
- }
- if len(list.Items) > 1 {
- return "", errors.Errorf("expected single statement, got %d statements", len(list.Items))
- }
- raw, ok := list.Items[0].(*oracleast.RawStmt)
- if !ok {
- return "", errors.Errorf("expected raw statement, got %T", list.Items[0])
- }
- selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)
- if !ok {
- return statement, nil
- }
-
- res, err := rewriteOracleSelectFetch(statement, selectStmt, limitCount)
- if err != nil {
- return "", err
- }
- // https://stackoverflow.com/questions/27987882/how-can-i-solve-ora-00911-invalid-character-error
- res = strings.TrimRightFunc(res, utils.IsSpaceOrSemicolon)
-
- return res, nil
-}
-
-func rewriteOracleSelectFetch(sql string, selectStmt *oracleast.SelectStmt, limitCount int) (string, error) {
- target := rightmostOracleSetSelect(selectStmt)
- if target.FetchFirst != nil {
- return rewriteOracleFetchClause(sql, target.FetchFirst, limitCount)
- }
- if target.ForUpdate != nil && target.ForUpdate.Loc.Start > 0 && target.ForUpdate.Loc.Start <= len(sql) {
- return sql[:target.ForUpdate.Loc.Start] + fmt.Sprintf("FETCH NEXT %d ROWS ONLY ", limitCount) + sql[target.ForUpdate.Loc.Start:], nil
- }
-
- loc := oracleast.NodeLoc(target)
- if loc.End < 0 || loc.End > len(sql) {
- return "", errors.Errorf("invalid SELECT end position %d", loc.End)
- }
- return sql[:loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[loc.End:], nil
-}
-
-func rightmostOracleSetSelect(selectStmt *oracleast.SelectStmt) *oracleast.SelectStmt {
- if selectStmt.Op != oracleast.SETOP_NONE && selectStmt.Rarg != nil {
- return rightmostOracleSetSelect(selectStmt.Rarg)
- }
- return selectStmt
-}
-
-func rewriteOracleFetchClause(sql string, fetch *oracleast.FetchFirstClause, limitCount int) (string, error) {
- if fetch.Count == nil {
- if fetch.Loc.Start < 0 || fetch.Loc.End < fetch.Loc.Start || fetch.Loc.End > len(sql) {
- return "", errors.Errorf("invalid FETCH position %d:%d", fetch.Loc.Start, fetch.Loc.End)
- }
- if hasOracleFetchKeyword(sql, fetch.Loc) {
- return sql, nil
- }
- return sql[:fetch.Loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[fetch.Loc.End:], nil
- }
- if fetch.Percent {
- return "", errors.Errorf("cannot rewrite PERCENT FETCH expression")
- }
-
- existingLimit := extractOracleFetchCount(fetch.Count)
- if existingLimit > 0 && existingLimit <= limitCount {
- return sql, nil
- }
-
- loc := oracleast.NodeLoc(fetch.Count)
- loc = trimOracleLocSpace(sql, loc)
- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
- if existingLimit <= 0 {
- return "", errors.Errorf("cannot rewrite non-constant FETCH expression")
- }
- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
- }
- return "", errors.Errorf("cannot rewrite FETCH expression")
-}
-
-func hasOracleFetchKeyword(sql string, loc oracleast.Loc) bool {
- segment := sql[loc.Start:loc.End]
- lexer := oracleparser.NewLexer(segment)
- for {
- tok := lexer.NextToken()
- if tok.Loc == tok.End && tok.End >= len(segment) {
- return false
- }
- if tok.Loc < 0 || tok.End > len(segment) || tok.End <= tok.Loc {
- return false
- }
- if strings.EqualFold(segment[tok.Loc:tok.End], "FETCH") {
- return true
- }
- }
-}
-
-func extractOracleFetchCount(node oracleast.Node) int {
- limit, ok := node.(*oracleast.NumberLiteral)
- if !ok || limit.IsFloat || limit.Ival <= 0 {
- return 0
- }
- return int(limit.Ival)
-}
-
-func trimOracleLocSpace(sql string, loc oracleast.Loc) oracleast.Loc {
- for loc.Start < loc.End && loc.Start < len(sql) && unicode.IsSpace(rune(sql[loc.Start])) {
- loc.Start++
- }
- for loc.End > loc.Start && loc.End <= len(sql) && unicode.IsSpace(rune(sql[loc.End-1])) {
- loc.End--
- }
- return loc
-}
-
-// ========== Skip Limit Logic for DUAL Queries ==========
-
-// skipAddLimit checks if the statement needs a limit clause.
-// For Oracle, we think the statement like "SELECT xxx FROM DUAL" does not need a limit clause.
-// More details, xxx can not be a subquery.
-func skipAddLimit(stmt string) (bool, error) {
- list, err := plsqlparser.ParsePLSQL(stmt)
- if err != nil {
- return false, err
- }
- if list == nil || len(list.Items) == 0 {
- return false, nil
- }
- // Multiple statements should not skip limit
- if len(list.Items) > 1 {
- return false, nil
- }
- raw, ok := list.Items[0].(*oracleast.RawStmt)
- if !ok {
- return false, nil
- }
- selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)
- if !ok {
- return false, nil
- }
- if !isSimpleOracleSelect(selectStmt) {
- return false, nil
- }
- if !isOracleSelectFromDual(selectStmt) {
- return false, nil
- }
- return !hasOracleSubqueriesInSelection(selectStmt), nil
-}
-
-func isSimpleOracleSelect(selectStmt *oracleast.SelectStmt) bool {
- return selectStmt.WithClause == nil &&
- !selectStmt.Distinct &&
- !selectStmt.UniqueKw &&
- !selectStmt.All &&
- selectStmt.Into == nil &&
- selectStmt.IntoVars == nil &&
- selectStmt.WhereClause == nil &&
- selectStmt.Hierarchical == nil &&
- selectStmt.GroupClause == nil &&
- selectStmt.HavingClause == nil &&
- selectStmt.ModelClause == nil &&
- len(selectStmt.WindowDefs) == 0 &&
- selectStmt.QualifyClause == nil &&
- selectStmt.OrderBy == nil &&
- selectStmt.ForUpdate == nil &&
- selectStmt.FetchFirst == nil &&
- selectStmt.Op == oracleast.SETOP_NONE
-}
-
-func isOracleSelectFromDual(selectStmt *oracleast.SelectStmt) bool {
- if selectStmt.FromClause == nil || len(selectStmt.FromClause.Items) != 1 {
- return false
- }
- tableRef, ok := selectStmt.FromClause.Items[0].(*oracleast.TableRef)
- if !ok || tableRef.Name == nil {
- return false
- }
- return tableRef.Alias == nil &&
- tableRef.Name.Schema == "" &&
- tableRef.Name.DBLink == "" &&
- strings.EqualFold(tableRef.Name.Name, "DUAL")
-}
-
-func hasOracleSubqueriesInSelection(selectStmt *oracleast.SelectStmt) bool {
- if selectStmt.TargetList == nil || len(selectStmt.TargetList.Items) == 0 {
- return true
- }
- for _, item := range selectStmt.TargetList.Items {
- target, ok := item.(*oracleast.ResTarget)
- if !ok || target.Expr == nil {
- return true
- }
- if _, ok := target.Expr.(*oracleast.Star); ok {
- return true
- }
- hasSubquery := false
- oracleast.Inspect(target.Expr, func(node oracleast.Node) bool {
- if _, ok := node.(*oracleast.SubqueryExpr); ok {
- hasSubquery = true
- return false
- }
- return true
- })
- if hasSubquery {
- return true
- }
- }
- return false
-}
diff --git a/backend/plugin/db/pg/pg.go b/backend/plugin/db/pg/pg.go
index 15fba29702..2e4c5b8627 100644
--- a/backend/plugin/db/pg/pg.go
+++ b/backend/plugin/db/pg/pg.go
@@ -805,7 +805,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_POSTGRES, statement, queryContext.Limit, "")
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_POSTGRES, statement)
diff --git a/backend/plugin/db/pg/query.go b/backend/plugin/db/pg/query.go
index c12bf5dd1e..f2b6f430e4 100644
--- a/backend/plugin/db/pg/query.go
+++ b/backend/plugin/db/pg/query.go
@@ -3,20 +3,15 @@ package pg
import (
"database/sql"
"fmt"
- "log/slog"
"strings"
"time"
- "github.com/pkg/errors"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/bytebase/omni/pg/ast"
"github.com/jackc/pgx/v5/pgtype"
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {
@@ -211,117 +206,3 @@ func padZeroes(rawStr string, acc int) string {
}
return rawStr
}
-
-// getStatementWithResultLimit returns the statement with LIMIT clause if not exists.
-func getStatementWithResultLimit(statement string, limit int) string {
- stmt, err := getStatementWithResultLimitInline(statement, limit)
- if err != nil {
- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- // Fallback to CTE approach for problematic queries
- return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)
- }
- return stmt
-}
-
-func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {
- if strings.TrimSpace(statement) == "" {
- return "", errors.New("empty statement")
- }
-
- stmts, err := pgparser.ParsePg(statement)
- if err != nil {
- return "", errors.Wrap(err, "failed to parse statement")
- }
-
- if len(stmts) != 1 {
- return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))
- }
-
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- // Non-SELECT statement, return as-is.
- return statement, nil
- }
-
- return rewriteSelectLimit(statement, sel, limitCount)
-}
-
-// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement
-// using byte-offset positions from the omni AST to surgically edit the original SQL text.
-func rewriteSelectLimit(sql string, sel *ast.SelectStmt, limitCount int) (string, error) {
- if sel.LimitCount != nil {
- // Already has LIMIT — replace the value if ours is lower.
- existingLimit := extractIntFromNode(sel.LimitCount)
- if existingLimit > 0 && existingLimit <= limitCount {
- return sql, nil // existing limit is already lower or equal, keep it
- }
- loc := nodeLocOf(sel.LimitCount)
- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
- }
- // LimitCount is a non-constant expression (e.g. LIMIT $1, LIMIT (1+2)).
- // Cannot safely rewrite in-place; let the caller fall back to CTE wrapper.
- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
- }
-
- // No LIMIT clause — find the right insertion point.
- // PostgreSQL grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...
- // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.
- insertPos, beforeLocking := findLimitInsertPosition(sel)
- if beforeLocking {
- // Inserting at the start of FOR UPDATE/SHARE. The original whitespace
- // before FOR becomes the separator before LIMIT; we add a trailing
- // space to separate the limit value from FOR.
- return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
- }
- return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
-}
-
-// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,
-// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).
-func findLimitInsertPosition(sel *ast.SelectStmt) (int, bool) {
- // LIMIT must appear before FOR UPDATE/SHARE.
- if sel.LockingClause != nil {
- span := ast.ListSpan(sel.LockingClause)
- if span.Start > 0 {
- return span.Start, true
- }
- }
-
- // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).
- // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,
- // so account for the sort clause explicitly.
- end := sel.Loc.End
- if span := ast.ListSpan(sel.SortClause); span.End > end {
- end = span.End
- }
- if end <= 0 {
- return 0, false
- }
- return end, false
-}
-
-// extractIntFromNode extracts an integer value from a LIMIT/OFFSET node.
-func extractIntFromNode(node ast.Node) int {
- switch n := node.(type) {
- case *ast.Integer:
- return int(n.Ival)
- case *ast.A_Const:
- if iv, ok := n.Val.(*ast.Integer); ok {
- return int(iv.Ival)
- }
- return 0
- default:
- return 0
- }
-}
-
-// nodeLocOf returns the Loc of a node, handling common wrapper types.
-func nodeLocOf(node ast.Node) ast.Loc {
- switch n := node.(type) {
- case *ast.A_Const:
- return n.Loc
- default:
- return ast.Loc{Start: -1, End: -1}
- }
-}
diff --git a/backend/plugin/db/pg/query_test.go b/backend/plugin/db/pg/query_test.go
index 282f9c0067..e8a3ed069e 100644
--- a/backend/plugin/db/pg/query_test.go
+++ b/backend/plugin/db/pg/query_test.go
@@ -1,16 +1,11 @@
package pg
import (
- "fmt"
- "strings"
"testing"
"time"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
-
- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg"
)
func TestBuildTimestamptzRowValue(t *testing.T) {
@@ -151,383 +146,3 @@ func TestPadZeroes(t *testing.T) {
require.Equal(t, test.expected, got)
}
}
-
-func TestGetStatementWithResultLimit(t *testing.T) {
- tests := []struct {
- name string
- statement string
- limit int
- want string
- }{
- {
- name: "Simple SELECT without LIMIT",
- statement: "SELECT * FROM users",
- limit: 10,
- want: "SELECT * FROM users LIMIT 10",
- },
- {
- name: "SELECT with WHERE clause",
- statement: "SELECT id, name FROM users WHERE active = true",
- limit: 5,
- want: "SELECT id, name FROM users WHERE active = true LIMIT 5",
- },
- {
- name: "SELECT with existing LIMIT higher than requested",
- statement: "SELECT * FROM users LIMIT 100",
- limit: 10,
- want: "SELECT * FROM users LIMIT 10",
- },
- {
- name: "SELECT with existing LIMIT lower than requested",
- statement: "SELECT * FROM users LIMIT 5",
- limit: 10,
- want: "SELECT * FROM users LIMIT 5",
- },
- {
- name: "WITH query (CTE)",
- statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",
- limit: 10,
- want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",
- },
- {
- name: "SELECT with ORDER BY",
- statement: "SELECT * FROM users ORDER BY created_at DESC",
- limit: 10,
- want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
- },
- {
- name: "SELECT with GROUP BY and HAVING",
- statement: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5",
- limit: 10,
- want: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5 LIMIT 10",
- },
- {
- name: "UNION query",
- statement: "SELECT id FROM users UNION SELECT id FROM admins",
- limit: 10,
- want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",
- },
- {
- name: "SELECT with OFFSET",
- statement: "SELECT * FROM users LIMIT 20 OFFSET 10",
- limit: 5,
- want: "SELECT * FROM users LIMIT 5 OFFSET 10",
- },
- {
- name: "SELECT with FOR UPDATE",
- statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
- limit: 10,
- want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
- },
- {
- name: "Non-SELECT statement (INSERT)",
- statement: "INSERT INTO users (name) VALUES ('John')",
- limit: 10,
- want: "INSERT INTO users (name) VALUES ('John')",
- },
- {
- name: "Non-SELECT statement (UPDATE)",
- statement: "UPDATE users SET active = false WHERE id = 1",
- limit: 10,
- want: "UPDATE users SET active = false WHERE id = 1",
- },
- {
- name: "Non-SELECT statement (DELETE)",
- statement: "DELETE FROM users WHERE active = false",
- limit: 10,
- want: "DELETE FROM users WHERE active = false",
- },
- {
- name: "SELECT with comments",
- statement: "SELECT * FROM users -- This is a comment\nWHERE active = true",
- limit: 10,
- want: "SELECT * FROM users -- This is a comment\nWHERE active = true LIMIT 10",
- },
- {
- name: "Complex nested SELECT",
- statement: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01'",
- limit: 10,
- want: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01' LIMIT 10",
- },
- {
- name: "SELECT with JOIN",
- statement: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id",
- limit: 10,
- want: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id LIMIT 10",
- },
- {
- name: "SELECT with parentheses",
- statement: "(SELECT * FROM users)",
- limit: 10,
- want: "(SELECT * FROM users) LIMIT 10",
- },
- {
- name: "SELECT with nested parentheses",
- statement: "((SELECT * FROM users))",
- limit: 10,
- want: "((SELECT * FROM users)) LIMIT 10",
- },
- {
- name: "SELECT with parentheses and existing LIMIT higher than requested",
- statement: "(SELECT * FROM users LIMIT 100)",
- limit: 10,
- want: "(SELECT * FROM users LIMIT 10)",
- },
- {
- name: "SELECT with parentheses and existing LIMIT lower than requested",
- statement: "(SELECT * FROM users LIMIT 5)",
- limit: 10,
- want: "(SELECT * FROM users LIMIT 5)",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := getStatementWithResultLimit(tt.statement, tt.limit)
-
- // For non-SELECT statements, the parser might fail and fall back to CTE approach
- // which we should avoid for non-SELECT statements
- if tt.statement == tt.want {
- // Non-SELECT statements should remain unchanged
- assert.Equal(t, tt.want, got)
- } else {
- // For SELECT statements, check if LIMIT was properly added
- // The exact format might vary slightly due to parser normalization
- assert.Contains(t, got, "LIMIT")
- assert.Contains(t, got, fmt.Sprintf("%d", tt.limit))
- }
- })
- }
-}
-
-func TestGetStatementWithResultLimitInline(t *testing.T) {
- tests := []struct {
- name string
- statement string
- limit int
- wantErr bool
- want string
- }{
- {
- name: "Valid SELECT statement",
- statement: "SELECT * FROM users",
- limit: 10,
- wantErr: false,
- want: "SELECT * FROM users LIMIT 10",
- },
- {
- name: "Invalid SQL syntax",
- statement: "SELECT * FROM WHERE",
- limit: 10,
- wantErr: true,
- },
- {
- name: "Empty statement",
- statement: "",
- limit: 10,
- wantErr: true,
- },
- {
- name: "FOR UPDATE gets proper whitespace",
- statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
- limit: 10,
- want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
- },
- {
- name: "FOR SHARE gets proper whitespace",
- statement: "SELECT * FROM users FOR SHARE",
- limit: 5,
- want: "SELECT * FROM users LIMIT 5 FOR SHARE",
- },
- {
- name: "SELECT ending with function call",
- statement: "SELECT now()",
- limit: 10,
- want: "SELECT now() LIMIT 10",
- },
- {
- name: "SELECT with subquery in WHERE",
- statement: "SELECT * FROM users WHERE id IN (SELECT id FROM admins)",
- limit: 10,
- want: "SELECT * FROM users WHERE id IN (SELECT id FROM admins) LIMIT 10",
- },
- {
- name: "Non-constant LIMIT expression falls back to error",
- statement: "SELECT * FROM t LIMIT (1+2)",
- limit: 5,
- wantErr: true,
- },
- {
- name: "CTE with lateral and ORDER BY",
- statement: `WITH params AS (
- SELECT 'resource.environment_id in []'::text AS env_condition
-), affected AS (
- SELECT
- p.resource,
- binding,
- COALESCE(binding->'condition'->>'expression', '') AS old_expression
- FROM policy p,
- LATERAL jsonb_array_elements(p.payload->'bindings') AS binding
- WHERE p.type = 'IAM'
- AND p.resource_type = 'PROJECT'
- AND binding->>'role' = 'roles/projectOwner'
- AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'
-)
-SELECT
- resource,
- binding->'members' AS members,
- binding->'condition' AS old_condition,
- CASE
- WHEN old_expression = '' THEN params.env_condition
- ELSE '(' || old_expression || ') && ' || params.env_condition
- END AS new_expression
-FROM affected
-CROSS JOIN params
-ORDER BY resource`,
- limit: 1000,
- want: `WITH params AS (
- SELECT 'resource.environment_id in []'::text AS env_condition
-), affected AS (
- SELECT
- p.resource,
- binding,
- COALESCE(binding->'condition'->>'expression', '') AS old_expression
- FROM policy p,
- LATERAL jsonb_array_elements(p.payload->'bindings') AS binding
- WHERE p.type = 'IAM'
- AND p.resource_type = 'PROJECT'
- AND binding->>'role' = 'roles/projectOwner'
- AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'
-)
-SELECT
- resource,
- binding->'members' AS members,
- binding->'condition' AS old_condition,
- CASE
- WHEN old_expression = '' THEN params.env_condition
- ELSE '(' || old_expression || ') && ' || params.env_condition
- END AS new_expression
-FROM affected
-CROSS JOIN params
-ORDER BY resource LIMIT 1000`,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getStatementWithResultLimitInline(tt.statement, tt.limit)
- if tt.wantErr {
- require.Error(t, err)
- } else {
- require.NoError(t, err)
- assert.NotEmpty(t, got)
- if tt.want != "" {
- assert.Equal(t, tt.want, got)
- }
- }
- })
- }
-}
-
-func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {
- tests := []struct {
- name string
- statement string
- limit int
- want string
- clausesInOrder []string
- }{
- {
- name: "ORDER BY",
- statement: "SELECT * FROM users ORDER BY created_at DESC",
- limit: 10,
- want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
- clausesInOrder: []string{"ORDER BY", "LIMIT 10"},
- },
- {
- name: "OFFSET without LIMIT",
- statement: "SELECT * FROM users OFFSET 20",
- limit: 10,
- clausesInOrder: []string{"OFFSET 20", "LIMIT 10"},
- },
- {
- name: "ORDER BY with OFFSET without LIMIT",
- statement: "SELECT * FROM users ORDER BY created_at DESC OFFSET 20",
- limit: 10,
- clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10"},
- },
- {
- name: "ORDER BY with FOR UPDATE",
- statement: "SELECT * FROM users ORDER BY id FOR UPDATE",
- limit: 10,
- want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",
- clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},
- },
- {
- name: "ORDER BY with OFFSET and FOR UPDATE",
- statement: "SELECT * FROM users ORDER BY id OFFSET 20 FOR UPDATE",
- limit: 10,
- clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10", "FOR UPDATE"},
- },
- {
- name: "GROUP BY HAVING WINDOW ORDER BY",
- statement: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC",
- limit: 10,
- want: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC LIMIT 10",
- clausesInOrder: []string{"GROUP BY", "HAVING", "WINDOW", "ORDER BY n", "LIMIT 10"},
- },
- {
- name: "UNION with outer ORDER BY",
- statement: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id",
- limit: 10,
- want: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id LIMIT 10",
- clausesInOrder: []string{"UNION ALL", "ORDER BY", "LIMIT 10"},
- },
- {
- name: "ORDER BY inside window only",
- statement: "SELECT row_number() OVER (ORDER BY created_at) FROM users",
- limit: 10,
- want: "SELECT row_number() OVER (ORDER BY created_at) FROM users LIMIT 10",
- clausesInOrder: []string{"FROM users", "LIMIT 10"},
- },
- {
- name: "ORDER BY inside subquery only",
- statement: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users",
- limit: 10,
- want: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users LIMIT 10",
- clausesInOrder: []string{"AS ordered_users", "LIMIT 10"},
- },
- {
- name: "WITH query with outer ORDER BY",
- statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",
- limit: 10,
- want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",
- clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getStatementWithResultLimitInline(tt.statement, tt.limit)
- require.NoError(t, err)
- if tt.want != "" {
- assert.Equal(t, tt.want, got)
- }
- assertSubstringsInOrder(t, got, tt.clausesInOrder)
-
- _, err = pgparser.ParsePg(got)
- require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)
- })
- }
-}
-
-func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {
- t.Helper()
-
- start := 0
- for _, substring := range substrings {
- index := strings.Index(s[start:], substring)
- require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)
- start += index + len(substring)
- }
-}
diff --git a/backend/plugin/db/redshift/redshift.go b/backend/plugin/db/redshift/redshift.go
index 751d4b7b47..6e86a7165a 100644
--- a/backend/plugin/db/redshift/redshift.go
+++ b/backend/plugin/db/redshift/redshift.go
@@ -394,7 +394,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_REDSHIFT, statement, queryContext.Limit, "")
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_REDSHIFT, statement)
@@ -447,7 +447,3 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
return results, nil
}
-
-func getStatementWithResultLimit(stmt string, limit int) string {
- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(stmt), limit)
-}
diff --git a/backend/plugin/db/snowflake/snowflake.go b/backend/plugin/db/snowflake/snowflake.go
index 92df80531c..2145101898 100644
--- a/backend/plugin/db/snowflake/snowflake.go
+++ b/backend/plugin/db/snowflake/snowflake.go
@@ -346,7 +346,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_SNOWFLAKE, statement, queryContext.Limit, "")
}
_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_SNOWFLAKE, statement)
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.go
index cd85749c63..148f93a1da 100644
--- a/backend/plugin/db/spanner/spanner.go
+++ b/backend/plugin/db/spanner/spanner.go
@@ -18,8 +18,6 @@ import (
spannerdb "cloud.google.com/go/spanner/admin/database/apiv1"
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
- "github.com/bytebase/omni/googlesql/ast"
- "github.com/bytebase/omni/googlesql/parser"
"github.com/pkg/errors"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
@@ -35,6 +33,9 @@ import (
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
+
+ // Register how this engine plans a limit, for base.StatementWithResultLimit below.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/spanner"
)
var (
@@ -304,32 +305,6 @@ func getColumnTypeName(columnType *sppb.Type) (string, error) {
return columnType.Code.String(), nil
}
-// getStatementWithResultLimit sets the LIMIT of the outermost query in stmt to
-// at most limit. It never wraps stmt in a subquery, where Spanner rejects a
-// WITH clause, so a query it cannot rewrite runs unchanged and queryStatement
-// caps its rows.
-func getStatementWithResultLimit(stmt string, limit int) string {
- file, errs := parser.Parse(stmt)
- if len(errs) > 0 || len(file.Stmts) != 1 {
- return stmt
- }
- query, ok := file.Stmts[0].(*ast.QueryStmt)
- // FOR UPDATE fails in queryStatement's read-only transaction, so there is no
- // point fitting a LIMIT before it.
- if !ok || query.ForUpdate {
- return stmt
- }
- if query.Limit == nil {
- end := query.Loc.End
- return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]
- }
- count, ok := query.Limit.(*ast.Literal)
- if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {
- return stmt
- }
- return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]
-}
-
// instancePath returns the Spanner instance resource name,
// e.g. projects/<project>/instances/<instance>.
func (d *Driver) instancePath() string {
@@ -372,7 +347,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
queryResult, err := func() (*v1pb.QueryResult, error) {
if util.IsSelect(statement) {
if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_SPANNER, statement, queryContext.Limit, "")
}
return d.queryStatement(ctx, statement, queryContext)
}
diff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.go
index 1e378453ba..7d6a56a9f0 100644
--- a/backend/plugin/db/spanner/spanner_test.go
+++ b/backend/plugin/db/spanner/spanner_test.go
@@ -110,80 +110,6 @@ func TestQueryConnLimitsQueryStartingWithWith(t *testing.T) {
}
}
-func TestGetStatementWithResultLimit(t *testing.T) {
- tests := []struct {
- name string
- statement string
- want string
- }{
- {
- name: "select",
- statement: "SELECT * FROM t",
- want: "SELECT * FROM t LIMIT 100",
- },
- {
- name: "with",
- statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",
- want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",
- },
- {
- name: "lowercase with, order by, and trailing semicolon",
- statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",
- want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",
- },
- {
- name: "with and set operation",
- statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",
- want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",
- },
- {
- name: "set operation",
- statement: "SELECT 1 UNION ALL SELECT 2",
- want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",
- },
- {
- name: "inner limit belongs to the parenthesized query",
- statement: "(SELECT x FROM t LIMIT 500)",
- want: "(SELECT x FROM t LIMIT 500) LIMIT 100",
- },
- {
- name: "trailing comment",
- statement: "SELECT x FROM t -- note",
- want: "SELECT x FROM t LIMIT 100 -- note",
- },
- {
- name: "lower limit is kept",
- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
- },
- {
- name: "higher limit is lowered",
- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",
- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",
- },
- {
- name: "non-literal limit is unchanged",
- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
- },
- {
- name: "for update is unchanged",
- statement: "SELECT x FROM t ORDER BY x FOR UPDATE",
- want: "SELECT x FROM t ORDER BY x FOR UPDATE",
- },
- {
- name: "unparsable query is unchanged",
- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
- },
- }
- for _, tc := range tests {
- t.Run(tc.name, func(t *testing.T) {
- require.Equal(t, tc.want, getStatementWithResultLimit(tc.statement, 100))
- })
- }
-}
-
func TestGetDatabaseFromDSN(t *testing.T) {
tests := []struct {
dsn string
diff --git a/backend/plugin/db/starrocks/query.go b/backend/plugin/db/starrocks/query.go
index c91d819d65..cde579313f 100644
--- a/backend/plugin/db/starrocks/query.go
+++ b/backend/plugin/db/starrocks/query.go
@@ -2,16 +2,7 @@ package starrocks
import (
"database/sql"
- "fmt"
- "log/slog"
- "strconv"
- "strings"
- "github.com/bytebase/omni/starrocks/ast"
- "github.com/bytebase/omni/starrocks/parser"
- "github.com/pkg/errors"
-
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
)
@@ -83,206 +74,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.
}
return util.NullRowValue
}
-
-func getStatementWithResultLimit(statement string, limit int) string {
- trimmedStatement := strings.TrimSpace(statement)
- if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {
- return statement
- }
-
- stmt, err := getStatementWithResultLimitInline(statement, limit)
- if err != nil {
- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)
- }
- return stmt
-}
-
-func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {
- if strings.TrimSpace(statement) == "" {
- return "", errors.New("empty statement")
- }
-
- file, errs := parser.Parse(statement)
- if len(errs) > 0 {
- return "", errors.New(errs[0].Error())
- }
- if file == nil || len(file.Stmts) != 1 {
- stmtCount := 0
- if file != nil {
- stmtCount = len(file.Stmts)
- }
- return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)
- }
-
- switch stmt := file.Stmts[0].(type) {
- case *ast.SelectStmt:
- return rewriteSelectLimit(statement, stmt, limitCount)
- case *ast.SetOpStmt:
- return rewriteSetOpLimit(statement, stmt, limitCount)
- case *ast.ParenSelect:
- // The wrapper itself may carry the trailing clauses —
- // (SELECT 1) LIMIT 900000 attaches the LIMIT to the ParenSelect,
- // not the inner query — so the wrapper is where the cap applies.
- if stmt.Limit != nil {
- return rewriteExistingLimit(statement, stmt.Limit, limitCount)
- }
- // No wrapper LIMIT: append one at the wrapper level, which covers
- // any paren nesting depth — ((SELECT 1)) LIMIT n and
- // (SELECT 1) ORDER BY 1 LIMIT n are engine-valid — instead of
- // unwrapping and missing deeper layers.
- if hasUnparsedTail(statement, stmt.Loc.End) {
- return "", errors.New("statement has unparsed tail content")
- }
- return statement[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + statement[stmt.Loc.End:], nil
- default:
- return statement, nil
- }
-}
-
-func hasUnparsedTail(sql string, locEnd int) bool {
- pos := locEnd
- for pos < len(sql) {
- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
- pos++
- }
- for pos < len(sql) && sql[pos] == ';' {
- pos++
- }
- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
- pos++
- }
- if pos >= len(sql) {
- return false
- }
- if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
- nl := strings.IndexByte(sql[pos:], '\n')
- if nl < 0 {
- return false
- }
- pos += nl + 1
- continue
- }
- if strings.HasPrefix(sql[pos:], "/*") {
- end := strings.Index(sql[pos:], "*/")
- if end < 0 {
- return false
- }
- pos += end + 2
- continue
- }
- return true
- }
- return false
-}
-
-// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it
-// is a constant no larger than limitCount, replace the count otherwise.
-func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {
- limitLoc := literalLoc(limitNode)
- if limitLoc.Start < 0 {
- return "", errors.New("cannot rewrite non-constant LIMIT expression")
- }
-
- if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {
- existingCount, _ := strconv.Atoi(sql[countStart:countEnd])
- if existingCount > 0 && existingCount <= limitCount {
- return sql, nil
- }
- return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil
- }
-
- existingLimit := extractLimitValue(limitNode)
- if existingLimit >= 0 && existingLimit <= limitCount {
- return sql, nil
- }
- return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil
-}
-
-func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
- if stmt.Limit != nil {
- return rewriteExistingLimit(sql, stmt.Limit, limitCount)
- }
-
- if stmt.Into != nil {
- return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil
- }
- if hasUnparsedTail(sql, stmt.Loc.End) {
- return "", errors.New("statement has unparsed tail content")
- }
- return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
-}
-
-func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {
- if stmt.Limit != nil {
- return rewriteExistingLimit(sql, stmt.Limit, limitCount)
- }
- if hasUnparsedTail(sql, stmt.Loc.End) {
- return "", errors.New("statement has unparsed tail content")
- }
- return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
-}
-
-func extractLimitValue(node ast.Node) int {
- lit, ok := node.(*ast.Literal)
- if !ok || lit.Kind != ast.LitInt {
- return -1
- }
- val, err := strconv.Atoi(lit.Value)
- if err != nil {
- return -1
- }
- return val
-}
-
-func literalLoc(node ast.Node) ast.Loc {
- lit, ok := node.(*ast.Literal)
- if !ok {
- return ast.Loc{Start: -1, End: -1}
- }
- return lit.Loc
-}
-
-func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {
- pos := limitLoc.End
- pos = skipWhitespaceAndComments(sql, pos)
- if pos >= len(sql) || sql[pos] != ',' {
- return 0, 0, false
- }
- pos++
- pos = skipWhitespaceAndComments(sql, pos)
- countStart = pos
- for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {
- pos++
- }
- if pos == countStart {
- return 0, 0, false
- }
- return countStart, pos, true
-}
-
-func skipWhitespaceAndComments(sql string, pos int) int {
- for pos < len(sql) {
- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
- pos++
- }
- if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
- nl := strings.IndexByte(sql[pos:], '\n')
- if nl < 0 {
- return len(sql)
- }
- pos += nl + 1
- continue
- }
- if strings.HasPrefix(sql[pos:], "/*") {
- end := strings.Index(sql[pos:], "*/")
- if end < 0 {
- return len(sql)
- }
- pos += end + 2
- continue
- }
- return pos
- }
- return pos
-}
diff --git a/backend/plugin/db/starrocks/starrocks.go b/backend/plugin/db/starrocks/starrocks.go
index f27436ce85..a3a1a3844c 100644
--- a/backend/plugin/db/starrocks/starrocks.go
+++ b/backend/plugin/db/starrocks/starrocks.go
@@ -315,7 +315,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(d.dbType, statement, queryContext.Limit, "")
}
sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)
diff --git a/backend/plugin/db/tidb/query.go b/backend/plugin/db/tidb/query.go
index 2090b2e1b7..b73fbbe361 100644
--- a/backend/plugin/db/tidb/query.go
+++ b/backend/plugin/db/tidb/query.go
@@ -2,20 +2,9 @@ package tidb
import (
"database/sql"
- "fmt"
- "log/slog"
- "strings"
- "github.com/pkg/errors"
-
- tidbast "github.com/pingcap/tidb/pkg/parser/ast"
- "github.com/pingcap/tidb/pkg/parser/format"
- tidbdriver "github.com/pingcap/tidb/pkg/types/parser_driver"
-
- "github.com/bytebase/bytebase/backend/common/log"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db/util"
- tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb"
)
func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {
@@ -85,68 +74,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.
}
return util.NullRowValue
}
-
-func getStatementWithResultLimit(statement string, limit int) string {
- stmt, err := getStatementWithResultLimitInline(statement, limit)
- if err != nil {
- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)
- }
- return stmt
-}
-
-func getStatementWithResultLimitInline(statement string, limit int) (string, error) {
- stmtList, err := tidbparser.ParseTiDB(statement, "", "")
- if err != nil {
- return "", errors.Wrapf(err, "failed to parse tidb statement: %s", statement)
- }
- if len(stmtList) != 1 {
- return "", errors.Errorf("expect one single statement in the query, %s", statement)
- }
- restoreFlags := format.DefaultRestoreFlags | format.RestoreStringWithoutDefaultCharset
- stmt := stmtList[0]
- switch stmt := stmt.(type) {
- case *tidbast.SelectStmt:
- if stmt.Limit != nil && stmt.Limit.Count != nil {
- if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {
- userLimit := int(v.GetInt64())
- if limit < userLimit {
- userLimit = limit
- }
- stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")
- }
- } else {
- stmt.Limit = &tidbast.Limit{
- Count: tidbast.NewValueExpr(int64(limit), "", ""),
- }
- }
- var buffer strings.Builder
- ctx := format.NewRestoreCtx(restoreFlags, &buffer)
- if err := stmt.Restore(ctx); err != nil {
- return "", err
- }
- return buffer.String(), nil
- case *tidbast.SetOprStmt:
- if stmt.Limit != nil && stmt.Limit.Count != nil {
- if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {
- userLimit := int(v.GetInt64())
- if limit < userLimit {
- userLimit = limit
- }
- stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")
- }
- } else {
- stmt.Limit = &tidbast.Limit{
- Count: tidbast.NewValueExpr(int64(limit), "", ""),
- }
- }
- var buffer strings.Builder
- ctx := format.NewRestoreCtx(restoreFlags, &buffer)
- if err := stmt.Restore(ctx); err != nil {
- return "", err
- }
- return buffer.String(), nil
- default:
- }
- return statement, nil
-}
diff --git a/backend/plugin/db/tidb/tidb.go b/backend/plugin/db/tidb/tidb.go
index e8e381811e..5066982488 100644
--- a/backend/plugin/db/tidb/tidb.go
+++ b/backend/plugin/db/tidb/tidb.go
@@ -379,7 +379,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string
}
statement = explained
} else if queryContext.Limit > 0 {
- statement = getStatementWithResultLimit(statement, queryContext.Limit)
+ statement = base.StatementWithResultLimit(storepb.Engine_TIDB, statement, queryContext.Limit, "")
}
sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)
diff --git a/backend/plugin/db/util/driverutil.go b/backend/plugin/db/util/driverutil.go
index 9f30d1ffb2..b9a0a1bb03 100644
--- a/backend/plugin/db/util/driverutil.go
+++ b/backend/plugin/db/util/driverutil.go
@@ -202,7 +202,7 @@ func ConvertCommonValue(_ string, _ *sql.ColumnType, value any) *v1pb.RowValue {
return NullRowValue
}
-// TrimStatement trims the unused characters from the statement for making getStatementWithResultLimit() happy.
+// TrimStatement trims the unused characters from a statement before running it.
func TrimStatement(statement string) string {
return strings.TrimLeftFunc(strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon), unicode.IsSpace)
}
diff --git a/backend/plugin/parser/base/limit.go b/backend/plugin/parser/base/limit.go
new file mode 100644
index 0000000000..5f85537f11
--- /dev/null
+++ b/backend/plugin/parser/base/limit.go
@@ -0,0 +1,58 @@
+package base
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+ "unicode"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/utils"
+)
+
+// ResultLimitFunc returns statement rewritten so its outermost query returns at
+// most limit rows. engineVersion is the connected server's version string; every
+// engine but Oracle ignores it (Oracle's clause depends on it — ROWNUM before
+// version 12, FETCH NEXT from 12c on).
+//
+// A function here never fails outward: each implementation tries a precise
+// rewrite and falls back to a wrapping subquery on any error, logging why. That
+// mirrors every registered function's actual shape, so the registry does not
+// carry an error return only every driver would immediately discard.
+type ResultLimitFunc func(statement string, limit int, engineVersion string) string
+
+var (
+ limitMux sync.Mutex
+ limitFuncs = make(map[storepb.Engine]ResultLimitFunc)
+)
+
+// RegisterResultLimitFunc registers how engine caps a statement's row count.
+// Only an engine whose driver enforces the cap in the SQL text registers here;
+// an engine with no entry runs the statement unchanged (see StatementWithResultLimit).
+func RegisterResultLimitFunc(engine storepb.Engine, f ResultLimitFunc) {
+ limitMux.Lock()
+ defer limitMux.Unlock()
+ if _, dup := limitFuncs[engine]; dup {
+ panic(fmt.Sprintf("Register called twice %s", engine))
+ }
+ limitFuncs[engine] = f
+}
+
+// StatementWithResultLimit returns statement rewritten to return at most limit
+// rows, for engines that enforce the cap in SQL text. An engine with nothing
+// registered — because its driver caps rows another way (Hive trims by result
+// size; MongoDB, Redis and the other non-SQL stores have no row cap here at
+// all) — returns statement unchanged; callers do not need to ask first.
+func StatementWithResultLimit(engine storepb.Engine, statement string, limit int, engineVersion string) string {
+ f, ok := limitFuncs[engine]
+ if !ok {
+ return statement
+ }
+ return f(statement, limit, engineVersion)
+}
+
+// TrimStatement trims the whitespace and trailing semicolons a limit rewrite's
+// fallback wrapper (a subquery or CTE) would otherwise leave stray inside itself.
+func TrimStatement(statement string) string {
+ return strings.TrimLeftFunc(strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon), unicode.IsSpace)
+}
diff --git a/backend/plugin/parser/base/limit_test.go b/backend/plugin/parser/base/limit_test.go
new file mode 100644
index 0000000000..e384d08d90
--- /dev/null
+++ b/backend/plugin/parser/base/limit_test.go
@@ -0,0 +1,46 @@
+package base
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+func TestStatementWithResultLimit(t *testing.T) {
+ // An engine with nothing registered — MongoDB, Redis, and the other stores
+ // whose drivers never call this — runs the statement unchanged rather than
+ // erroring, unlike ExplainStatement: there is no "must refuse" case here.
+ got := StatementWithResultLimit(storepb.Engine_MONGODB, "SELECT * FROM t", 10, "")
+ require.Equal(t, "SELECT * FROM t", got)
+
+ // PostgreSQL registers in this test binary via the parser/pg package's own
+ // tests importing this one transitively is not guaranteed, so this only
+ // covers the registry mechanics directly with a throwaway engine value.
+}
+
+func TestRegisterResultLimitFuncPanicsOnDuplicate(t *testing.T) {
+ const engine = storepb.Engine(999999) // not a real engine; scoped to this test
+ RegisterResultLimitFunc(engine, func(statement string, _ int, _ string) string { return statement })
+ require.Panics(t, func() {
+ RegisterResultLimitFunc(engine, func(statement string, _ int, _ string) string { return statement })
+ })
+}
+
+func TestTrimStatement(t *testing.T) {
+ tests := []struct {
+ statement string
+ want string
+ }{
+ {"SELECT 1", "SELECT 1"},
+ {"SELECT 1;", "SELECT 1"},
+ {"SELECT 1;;;", "SELECT 1"},
+ {" SELECT 1 ", "SELECT 1"},
+ {"\n\tSELECT 1\n", "SELECT 1"},
+ {"", ""},
+ }
+ for _, tc := range tests {
+ require.Equalf(t, tc.want, TrimStatement(tc.statement), "%q", tc.statement)
+ }
+}
diff --git a/backend/plugin/parser/bigquery/limit.go b/backend/plugin/parser/bigquery/limit.go
new file mode 100644
index 0000000000..cd746f112b
--- /dev/null
+++ b/backend/plugin/parser/bigquery/limit.go
@@ -0,0 +1,23 @@
+package bigquery
+
+import (
+ "fmt"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_BIGQUERY, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for BigQuery, which
+// takes no engineVersion. The caller only applies this to a statement it has
+// already confirmed is a SELECT (see bigquery.go), so this wraps unconditionally.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ limitPart := ""
+ if limit > 0 {
+ limitPart = fmt.Sprintf(" LIMIT %d", limit)
+ }
+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", base.TrimStatement(statement), limitPart)
+}
diff --git a/backend/plugin/parser/bigquery/limit_test.go b/backend/plugin/parser/bigquery/limit_test.go
new file mode 100644
index 0000000000..7264df875f
--- /dev/null
+++ b/backend/plugin/parser/bigquery/limit_test.go
@@ -0,0 +1,23 @@
+package bigquery
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ statement string
+ limit int
+ want string
+ }{
+ {"SELECT * FROM t", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},
+ {"SELECT * FROM t;", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},
+ {"SELECT * FROM t", 0, "WITH result AS (SELECT * FROM t) SELECT * FROM result;"},
+ }
+ for _, tc := range tests {
+ got := statementWithResultLimit(tc.statement, tc.limit, "")
+ require.Equalf(t, tc.want, got, "%q", tc.statement)
+ }
+}
diff --git a/backend/plugin/parser/doris/limit.go b/backend/plugin/parser/doris/limit.go
new file mode 100644
index 0000000000..22490a5249
--- /dev/null
+++ b/backend/plugin/parser/doris/limit.go
@@ -0,0 +1,242 @@
+package doris
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ "github.com/bytebase/omni/doris/ast"
+ "github.com/bytebase/omni/doris/parser"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_DORIS, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for Doris, which
+// takes no engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ trimmedStatement := strings.TrimSpace(statement)
+ if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {
+ return statement
+ }
+
+ query, outfile := splitOutfileClause(statement)
+ stmt, err := statementWithResultLimitInline(query, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ if outfile == "" {
+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(query), limit)
+ }
+ stmt = fmt.Sprintf("SELECT * FROM (\n%s\n) result LIMIT %d", base.TrimStatement(query), limit)
+ }
+ if outfile != "" {
+ return strings.TrimRight(stmt, " \t\r\n") + "\n" + outfile
+ }
+ return stmt
+}
+
+// The Doris AST cannot represent INTO OUTFILE. Keep this top-level clause outside
+// the query while capping it, including when a set operation needs wrapping.
+func splitOutfileClause(statement string) (string, string) {
+ tokens, errs := parser.Tokenize(statement)
+ if len(errs) > 0 {
+ return statement, ""
+ }
+ intoKind, _ := parser.KeywordToken("into")
+ depth := 0
+ for i, token := range tokens {
+ switch token.Kind {
+ case '(':
+ depth++
+ case ')':
+ depth--
+ case ';':
+ return statement, ""
+ case intoKind:
+ if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileTokenKind {
+ return statement[:token.Loc.Start], statement[token.Loc.Start:]
+ }
+ default:
+ }
+ }
+ return statement, ""
+}
+
+func statementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ file, errs := parser.Parse(statement)
+ if len(errs) > 0 {
+ return "", errors.New(errs[0].Error())
+ }
+ if file == nil || len(file.Stmts) != 1 {
+ stmtCount := 0
+ if file != nil {
+ stmtCount = len(file.Stmts)
+ }
+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)
+ }
+
+ switch stmt := file.Stmts[0].(type) {
+ case *ast.SelectStmt:
+ return rewriteSelectLimit(statement, stmt, limitCount)
+ case *ast.SetOpStmt:
+ // SetOpStmt.Limit is unusable: the Doris grammar leaves it nil whether or
+ // not the text has a trailing LIMIT (confirmed against omni — a UNION ...
+ // LIMIT 5 parses with Limit == nil, its text still inside the node's own
+ // Loc). Trusting nil to mean "no LIMIT" would append a second one after
+ // the caller's. This is an error, not a pass-through: a set operation is
+ // still a query whose rows need capping, so the caller's CTE-wrap
+ // fallback must run rather than let it through unbounded.
+ return "", errors.New("cannot locate LIMIT on a Doris set operation")
+ default:
+ // A statement Doris parses to something other than a query — DDL, DML,
+ // SHOW, etc. — has no rows to cap; run it unchanged, matching every other
+ // engine's treatment of a non-SELECT here.
+ return statement, nil
+ }
+}
+
+func hasUnparsedTail(sql string, locEnd int) bool {
+ pos := locEnd
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ for pos < len(sql) && sql[pos] == ';' {
+ pos++
+ }
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if pos >= len(sql) {
+ return false
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return false
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return false
+ }
+ pos += end + 2
+ continue
+ }
+ return true
+ }
+ return false
+}
+
+// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it
+// is a constant no larger than limitCount, replace the count otherwise.
+func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {
+ limitLoc := literalLoc(limitNode)
+ if limitLoc.Start < 0 {
+ return "", errors.New("cannot rewrite non-constant LIMIT expression")
+ }
+
+ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {
+ // A literal 0 count (LIMIT offset,0) is a valid, stricter limit and must
+ // be kept, matching the non-comma path below.
+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])
+ if existingCount >= 0 && existingCount <= limitCount {
+ return sql, nil
+ }
+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil
+ }
+
+ existingLimit := extractLimitValue(limitNode)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil
+}
+
+func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil {
+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)
+ }
+ if hasUnparsedTail(sql, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
+ }
+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
+}
+
+func extractLimitValue(node ast.Node) int {
+ lit, ok := node.(*ast.Literal)
+ if !ok || lit.Kind != ast.LitInt {
+ return -1
+ }
+ val, err := strconv.Atoi(lit.Value)
+ if err != nil {
+ return -1
+ }
+ return val
+}
+
+func literalLoc(node ast.Node) ast.Loc {
+ lit, ok := node.(*ast.Literal)
+ if !ok {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ return lit.Loc
+}
+
+func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {
+ pos := limitLoc.End
+ pos = skipWhitespaceAndComments(sql, pos)
+ if pos >= len(sql) || sql[pos] != ',' {
+ return 0, 0, false
+ }
+ pos++
+ pos = skipWhitespaceAndComments(sql, pos)
+ countStart = pos
+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {
+ pos++
+ }
+ if pos == countStart {
+ return 0, 0, false
+ }
+ return countStart, pos, true
+}
+
+func skipWhitespaceAndComments(sql string, pos int) int {
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return len(sql)
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return len(sql)
+ }
+ pos += end + 2
+ continue
+ }
+ return pos
+ }
+ return pos
+}
diff --git a/backend/plugin/parser/doris/limit_test.go b/backend/plugin/parser/doris/limit_test.go
new file mode 100644
index 0000000000..baa2d5ca43
--- /dev/null
+++ b/backend/plugin/parser/doris/limit_test.go
@@ -0,0 +1,263 @@
+package doris
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetStatementWithResultLimit(t *testing.T) {
+ testCases := []struct {
+ stmt string
+ count int
+ want string
+ }{
+ {
+ stmt: "SELECT * FROM t;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 10;",
+ count: 5,
+ want: "SELECT * FROM t LIMIT 5;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 5;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 5;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 123;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 10 OFFSET 123;",
+ count: 5,
+ want: "SELECT * FROM t LIMIT 5 OFFSET 123;",
+ },
+ {
+ stmt: "SELECT * FROM t2 JOIN t1 ON t2.c2 = t1.c2 where t2.c2 > 10;",
+ count: 10,
+ want: "SELECT * FROM t2 JOIN t1 ON t2.c2 = t1.c2 where t2.c2 > 10 LIMIT 10;",
+ },
+ {
+ stmt: "SELECT * FROM t1 ORDER BY c2;",
+ count: 10,
+ want: "SELECT * FROM t1 ORDER BY c2 LIMIT 10;",
+ },
+ {
+ stmt: "SELECT COUNT(col1) AS col2 FROM t GROUP BY col2 HAVING col2 = 2;",
+ count: 10,
+ want: "SELECT COUNT(col1) AS col2 FROM t GROUP BY col2 HAVING col2 = 2 LIMIT 10;",
+ },
+ {
+ stmt: "SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers;",
+ count: 10,
+ // SetOpStmt.Limit is unreliable in the Doris grammar (see limit.go),
+ // so a set operation always takes the CTE-wrap fallback.
+ want: "SELECT * FROM (SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers) result LIMIT 10;",
+ },
+ {
+ stmt: "SELECT customerNumber, checkNumber, amount FROM payments WHERE amount = (SELECT MAX(amount) FROM payments);",
+ count: 10,
+ want: "SELECT customerNumber, checkNumber, amount FROM payments WHERE amount = (SELECT MAX(amount) FROM payments) LIMIT 10;",
+ },
+ {
+ stmt: "SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers LIMIT 10;",
+ count: 10,
+ want: "SELECT * FROM (SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers LIMIT 10) result LIMIT 10;",
+ },
+ {
+ // The outer statement is a SelectStmt (its WITH clause), so it still
+ // gets the precise in-place edit despite the UNION ALL inside the CTE.
+ stmt: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count;",
+ count: 10,
+ want: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count LIMIT 10;",
+ },
+ {
+ stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",
+ count: 10,
+ want: "SELECT * FROM (SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers) result LIMIT 10;",
+ },
+ {
+ stmt: "SELECT col1, col2 FROM table1 ORDER BY col1;",
+ count: 10,
+ want: "SELECT col1, col2 FROM table1 ORDER BY col1 LIMIT 10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 0,20;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0,10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 123,20;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 123,10;",
+ },
+ {
+ // A literal 0 count is kept, not replaced with the requested cap.
+ stmt: "SELECT * FROM t LIMIT 5,0;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 5,0;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 0,\n1000000;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0,\n10;",
+ },
+ {
+ stmt: "SELECT * FROM t LIMIT 0 /* offset comment */, 1000000;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 0 /* offset comment */, 10;",
+ },
+ // SHOW statements should not be wrapped
+ {
+ stmt: "SHOW DATA",
+ count: 10,
+ want: "SHOW DATA",
+ },
+ {
+ stmt: "SHOW TABLES",
+ count: 10,
+ want: "SHOW TABLES",
+ },
+ {
+ stmt: "show data",
+ count: 10,
+ want: "show data",
+ },
+ {
+ stmt: " SHOW DATA ",
+ count: 10,
+ want: " SHOW DATA ",
+ },
+ {
+ stmt: "SELECT * FROM t -- note",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 -- note",
+ },
+ {
+ stmt: "SELECT * FROM t /* block comment */;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",
+ },
+ {
+ stmt: "SELECT * FROM t # hash comment",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10 # hash comment",
+ },
+ {
+ stmt: "SELECT * FROM t;\n",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 10;\n",
+ },
+ // A set operation always takes the CTE-wrap fallback (see limit.go), so an
+ // existing LIMIT on it — larger, smaller, or comma-form — is never capped
+ // in place; the fallback's own LIMIT is still the one that binds.
+ {
+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100;",
+ count: 10,
+ want: "SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100) result LIMIT 10;",
+ },
+ {
+ stmt: "SELECT a FROM t1 INTERSECT SELECT b FROM t2;",
+ count: 10,
+ want: "SELECT * FROM (SELECT a FROM t1 INTERSECT SELECT b FROM t2) result LIMIT 10;",
+ },
+ // Doris has no dedicated node for a parenthesized top-level query: the
+ // grammar strips the parens transparently and produces a plain
+ // SelectStmt whose Loc always lands after the closing paren(s), so a
+ // bare append at Loc.End still lands outside them and is valid SQL.
+ {
+ stmt: "(SELECT * FROM t)",
+ count: 10,
+ want: "(SELECT * FROM t) LIMIT 10",
+ },
+ {
+ stmt: "((SELECT * FROM t))",
+ count: 10,
+ want: "((SELECT * FROM t)) LIMIT 10",
+ },
+ {
+ stmt: "(SELECT * FROM t) LIMIT 1000000",
+ count: 10,
+ want: "(SELECT * FROM t) LIMIT 10",
+ },
+ {
+ stmt: "(SELECT * FROM t) ORDER BY 1",
+ count: 10,
+ want: "(SELECT * FROM t) ORDER BY 1 LIMIT 10",
+ },
+ }
+
+ for _, tc := range testCases {
+ got := statementWithResultLimit(tc.stmt, tc.count, "")
+ require.Equal(t, tc.want, got, tc.stmt)
+ }
+}
+
+func TestResultLimitOutfile(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want string
+ }{
+ {
+ name: "select with export options",
+ statement: `SELECT * FROM orders INTO OUTFILE 's3://bucket/export/' FORMAT AS CSV PROPERTIES("column_separator" = ",");`,
+ want: "SELECT * FROM orders LIMIT 10\n" + `INTO OUTFILE 's3://bucket/export/' FORMAT AS CSV PROPERTIES("column_separator" = ",");`,
+ },
+ {
+ name: "existing limit",
+ statement: "SELECT * FROM orders LIMIT 100 INTO OUTFILE 's3://bucket/export/';",
+ want: "SELECT * FROM orders LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "zero limit",
+ statement: "SELECT * FROM orders LIMIT 0 INTO OUTFILE 's3://bucket/export/';",
+ want: "SELECT * FROM orders LIMIT 0\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "zero comma limit",
+ statement: "SELECT * FROM orders LIMIT 5,0 INTO OUTFILE 's3://bucket/export/';",
+ want: "SELECT * FROM orders LIMIT 5,0\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "union fallback",
+ statement: "SELECT a FROM t1 UNION SELECT a FROM t2 INTO OUTFILE 's3://bucket/export/';",
+ want: "SELECT * FROM (\nSELECT a FROM t1 UNION SELECT a FROM t2\n) result LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "comments and case",
+ statement: "SELECT * FROM orders -- export\ninto /* target */ outfile 's3://bucket/export/';",
+ want: "SELECT * FROM orders LIMIT 10 -- export\ninto /* target */ outfile 's3://bucket/export/';",
+ },
+ {
+ name: "union with trailing comment",
+ statement: "SELECT a FROM t1 UNION SELECT a FROM t2 -- export\nINTO OUTFILE 's3://bucket/export/';",
+ want: "SELECT * FROM (\nSELECT a FROM t1 UNION SELECT a FROM t2 -- export\n) result LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "parenthesized query",
+ statement: "(SELECT * FROM orders) INTO OUTFILE 's3://bucket/export/';",
+ want: "(SELECT * FROM orders) LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",
+ },
+ {
+ name: "quoted words",
+ statement: "SELECT 'INTO OUTFILE', `INTO OUTFILE` FROM t",
+ want: "SELECT 'INTO OUTFILE', `INTO OUTFILE` FROM t LIMIT 10",
+ },
+ {
+ name: "comment words",
+ statement: "SELECT * FROM t /* INTO OUTFILE */;",
+ want: "SELECT * FROM t LIMIT 10 /* INTO OUTFILE */;",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.want, statementWithResultLimit(tt.statement, 10, ""))
+ })
+ }
+}
diff --git a/backend/plugin/parser/mysql/limit.go b/backend/plugin/parser/mysql/limit.go
new file mode 100644
index 0000000000..6388c41fec
--- /dev/null
+++ b/backend/plugin/parser/mysql/limit.go
@@ -0,0 +1,253 @@
+package mysql
+
+import (
+ "fmt"
+ "log/slog"
+ "reflect"
+ "strings"
+
+ "github.com/bytebase/omni/mysql/ast"
+ omnimysqlparser "github.com/bytebase/omni/mysql/parser"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_MYSQL, statementWithResultLimit)
+ base.RegisterResultLimitFunc(storepb.Engine_MARIADB, statementWithResultLimit)
+ base.RegisterResultLimitFunc(storepb.Engine_OCEANBASE, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for MySQL, MariaDB
+// and OceanBase, which share this grammar. None takes an engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := statementWithResultLimitInline(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ // MySQL 5.7 doesn't support WITH clause.
+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
+
+func statementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ list, err := ParseMySQL(statement)
+ if err != nil {
+ if stmt, procedureErr := rewriteMySQLSelectProcedureLimit(statement, limitCount); procedureErr == nil {
+ return stmt, nil
+ }
+ return "", err
+ }
+ if len(list.Items) != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", len(list.Items))
+ }
+
+ stmt, ok := list.Items[0].(*ast.SelectStmt)
+ if !ok {
+ return statement, nil
+ }
+
+ return rewriteMySQLSelectLimit(statement, stmt, limitCount)
+}
+
+func rewriteMySQLSelectProcedureLimit(sql string, limitCount int) (string, error) {
+ statements, err := SplitSQL(sql)
+ if err != nil {
+ return "", err
+ }
+ statementCount := 0
+ for _, statement := range statements {
+ if !statement.Empty {
+ statementCount++
+ }
+ }
+ if statementCount != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", statementCount)
+ }
+
+ tokens := omnimysqlparser.Tokenize(sql)
+ if len(tokens) == 0 || omnimysqlparser.TokenName(tokens[0].Type) != "SELECT" {
+ return "", errors.New("not a SELECT PROCEDURE statement")
+ }
+ for i, token := range tokens[1:] {
+ if omnimysqlparser.TokenName(token.Type) == "PROCEDURE" {
+ procedureTokenIndex := i + 1
+ if stmt, ok := rewriteMySQLLimitBeforeProcedure(sql, tokens[:procedureTokenIndex], limitCount); ok {
+ return stmt, nil
+ }
+ return sql[:token.Loc] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[token.Loc:], nil
+ }
+ }
+ return "", errors.New("SELECT PROCEDURE clause not found")
+}
+
+func rewriteMySQLLimitBeforeProcedure(sql string, tokens []omnimysqlparser.Token, limitCount int) (string, bool) {
+ depth := 0
+ for i := len(tokens) - 1; i >= 0; i-- {
+ switch tokens[i].Str {
+ case ")":
+ depth++
+ continue
+ case "(":
+ if depth > 0 {
+ depth--
+ }
+ continue
+ default:
+ }
+ if depth > 0 {
+ continue
+ }
+ if omnimysqlparser.TokenName(tokens[i].Type) != "LIMIT" {
+ continue
+ }
+ countTokenIndex := i + 1
+ if i+3 < len(tokens) && tokens[i+2].Str == "," {
+ countTokenIndex = i + 3
+ }
+ if countTokenIndex >= len(tokens) {
+ return "", false
+ }
+ if tokens[countTokenIndex].Ival > 0 && int(tokens[countTokenIndex].Ival) <= limitCount {
+ return sql, true
+ }
+ return sql[:tokens[countTokenIndex].Loc] + fmt.Sprintf("%d", limitCount) + sql[tokens[countTokenIndex].End:], true
+ }
+ return "", false
+}
+
+func rewriteMySQLSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil && stmt.Limit.Count != nil {
+ existingLimit := extractMySQLLimit(stmt.Limit.Count)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+ loc := nodeLocOf(stmt.Limit.Count)
+ loc = trimMySQLLocSpace(sql, loc)
+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
+ if existingLimit < 0 && stmt.Into == nil {
+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
+ }
+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
+ }
+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")
+ }
+
+ insertPos, beforeClause := findMySQLLimitInsertPosition(sql, stmt)
+ if insertPos < 0 || insertPos > len(sql) {
+ return "", errors.Errorf("invalid LIMIT insert position %d", insertPos)
+ }
+ if beforeClause {
+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
+ }
+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
+}
+
+func findMySQLLimitInsertPosition(sql string, stmt *ast.SelectStmt) (int, bool) {
+ if stmt.Into != nil && stmt.Into.Loc.Start > 0 {
+ intoStart := findMySQLKeywordBefore(sql, stmt.Into.Loc.Start, "INTO")
+ if intoStart >= maxMySQLLocEndBeforeTailClauses(stmt) {
+ return intoStart, true
+ }
+ }
+ if stmt.ForUpdate != nil && stmt.ForUpdate.Loc.Start > 0 {
+ return stmt.ForUpdate.Loc.Start, true
+ }
+ if stmt.Loc.End > 0 {
+ return stmt.Loc.End, false
+ }
+ return -1, false
+}
+
+func extractMySQLLimit(node ast.Node) int {
+ limit, ok := node.(*ast.IntLit)
+ if !ok {
+ return -1
+ }
+ return int(limit.Value)
+}
+
+func nodeLocOf(node ast.Node) ast.Loc {
+ return mysqlNodeLoc(node)
+}
+
+func trimMySQLLocSpace(sql string, loc ast.Loc) ast.Loc {
+ for loc.Start < loc.End && loc.Start < len(sql) && (sql[loc.Start] == ' ' || sql[loc.Start] == '\t' || sql[loc.Start] == '\n' || sql[loc.Start] == '\r') {
+ loc.Start++
+ }
+ for loc.End > loc.Start && loc.End <= len(sql) && (sql[loc.End-1] == ' ' || sql[loc.End-1] == '\t' || sql[loc.End-1] == '\n' || sql[loc.End-1] == '\r') {
+ loc.End--
+ }
+ return loc
+}
+
+func maxMySQLLocEndBeforeTailClauses(node ast.Node) int {
+ maxEnd := -1
+ ast.Inspect(node, func(n ast.Node) bool {
+ if n == nil {
+ return false
+ }
+ if n == node {
+ return true
+ }
+ if _, ok := n.(*ast.IntoClause); ok {
+ return false
+ }
+ if _, ok := n.(*ast.ForUpdate); ok {
+ return false
+ }
+ if loc := mysqlNodeLoc(n); loc.End > maxEnd {
+ maxEnd = loc.End
+ }
+ return true
+ })
+ return maxEnd
+}
+
+func mysqlNodeLoc(node ast.Node) ast.Loc {
+ if node == nil {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ value := reflect.ValueOf(node)
+ if value.Kind() != reflect.Pointer || value.IsNil() {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ elem := value.Elem()
+ if elem.Kind() != reflect.Struct {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ field := elem.FieldByName("Loc")
+ if !field.IsValid() || !field.CanInterface() {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ loc, ok := field.Interface().(ast.Loc)
+ if !ok {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ return loc
+}
+
+func findMySQLKeywordBefore(sql string, offset int, keyword string) int {
+ if offset > len(sql) {
+ offset = len(sql)
+ }
+ if len(keyword) == 0 {
+ return offset
+ }
+ keyword = strings.ToUpper(keyword)
+ tokens := omnimysqlparser.Tokenize(sql[:offset])
+ for i := len(tokens) - 1; i >= 0; i-- {
+ token := tokens[i]
+ if token.End <= offset && omnimysqlparser.TokenName(token.Type) == keyword {
+ return token.Loc
+ }
+ }
+ return offset
+}
diff --git a/backend/plugin/db/mysql/query_test.go b/backend/plugin/parser/mysql/limit_test.go
similarity index 97%
rename from backend/plugin/db/mysql/query_test.go
rename to backend/plugin/parser/mysql/limit_test.go
index e86281819a..a80d8f28e7 100644
--- a/backend/plugin/db/mysql/query_test.go
+++ b/backend/plugin/parser/mysql/limit_test.go
@@ -213,7 +213,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {
}
for _, tc := range testCases {
- got := getStatementWithResultLimit(tc.stmt, tc.count)
+ got := statementWithResultLimit(tc.stmt, tc.count, "")
require.Equal(t, tc.want, got, tc.stmt)
}
}
@@ -237,7 +237,7 @@ func TestGetStatementWithResultLimitFallback(t *testing.T) {
}
for _, tc := range testCases {
- got := getStatementWithResultLimit(tc.stmt, tc.count)
+ got := statementWithResultLimit(tc.stmt, tc.count, "")
require.Equal(t, tc.want, got, tc.stmt)
}
}
@@ -277,7 +277,7 @@ func TestGetStatementWithResultLimitInline(t *testing.T) {
}
for _, tc := range testCases {
- got, err := getStatementWithResultLimitInline(tc.stmt, tc.count)
+ got, err := statementWithResultLimitInline(tc.stmt, tc.count)
if tc.wantErr {
require.Error(t, err, tc.stmt)
continue
@@ -291,6 +291,6 @@ func TestGetStatementWithResultLimitUnicodeBeforeInto(t *testing.T) {
stmt := "SELECT 'ı' FROM t INTO OUTFILE '/tmp/a';"
want := "SELECT 'ı' FROM t LIMIT 10 INTO OUTFILE '/tmp/a';"
- got := getStatementWithResultLimit(stmt, 10)
+ got := statementWithResultLimit(stmt, 10, "")
require.Equal(t, want, got)
}
diff --git a/backend/plugin/parser/pg/limit.go b/backend/plugin/parser/pg/limit.go
new file mode 100644
index 0000000000..b8deb7f308
--- /dev/null
+++ b/backend/plugin/parser/pg/limit.go
@@ -0,0 +1,136 @@
+package pg
+
+import (
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/bytebase/omni/pg/ast"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_POSTGRES, statementWithResultLimit)
+ // CockroachDB parses through the PostgreSQL grammar here as it does for the
+ // read-only gate and EXPLAIN (see query.go, explain.go).
+ base.RegisterResultLimitFunc(storepb.Engine_COCKROACHDB, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for PostgreSQL and
+// CockroachDB, which share this grammar. Neither takes an engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := statementWithResultLimitInline(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ // Fallback to CTE approach for problematic queries
+ return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
+
+func statementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ stmts, err := ParsePg(statement)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to parse statement")
+ }
+
+ if len(stmts) != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))
+ }
+
+ sel, ok := stmts[0].AST.(*ast.SelectStmt)
+ if !ok {
+ // Non-SELECT statement, return as-is.
+ return statement, nil
+ }
+
+ return rewriteSelectLimit(statement, sel, limitCount)
+}
+
+// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement
+// using byte-offset positions from the omni AST to surgically edit the original SQL text.
+func rewriteSelectLimit(sql string, sel *ast.SelectStmt, limitCount int) (string, error) {
+ if sel.LimitCount != nil {
+ existingLimit, isInteger := extractIntFromNode(sel.LimitCount)
+ loc := nodeLocOf(sel.LimitCount)
+ if isInteger && existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil // existing limit is already lower or equal, keep it
+ }
+ if isInteger && loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
+ }
+ // ALL/NULL and non-constant limits lack reliable replacement spans.
+ // Cap them with the outer query instead of rewriting their text.
+ return "", errors.Errorf("cannot rewrite LIMIT expression in place")
+ }
+
+ // No LIMIT clause — find the right insertion point.
+ // PostgreSQL grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...
+ // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.
+ insertPos, beforeLocking := findLimitInsertPosition(sel)
+ if beforeLocking {
+ // Inserting at the start of FOR UPDATE/SHARE. The original whitespace
+ // before FOR becomes the separator before LIMIT; we add a trailing
+ // space to separate the limit value from FOR.
+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
+ }
+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
+}
+
+// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,
+// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).
+func findLimitInsertPosition(sel *ast.SelectStmt) (int, bool) {
+ // LIMIT must appear before FOR UPDATE/SHARE.
+ if sel.LockingClause != nil {
+ span := ast.ListSpan(sel.LockingClause)
+ if span.Start > 0 {
+ return span.Start, true
+ }
+ }
+
+ // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).
+ // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,
+ // so account for the sort clause explicitly.
+ end := sel.Loc.End
+ if span := ast.ListSpan(sel.SortClause); span.End > end {
+ end = span.End
+ }
+ if end <= 0 {
+ return 0, false
+ }
+ return end, false
+}
+
+// extractIntFromNode distinguishes integer limits, including zero, from unlimited
+// ALL/NULL constants and expressions that require the fallback wrapper.
+func extractIntFromNode(node ast.Node) (int, bool) {
+ switch n := node.(type) {
+ case *ast.Integer:
+ return int(n.Ival), true
+ case *ast.A_Const:
+ if iv, ok := n.Val.(*ast.Integer); ok {
+ return int(iv.Ival), true
+ }
+ return 0, false
+ default:
+ return 0, false
+ }
+}
+
+// nodeLocOf returns the Loc of a node, handling common wrapper types.
+func nodeLocOf(node ast.Node) ast.Loc {
+ switch n := node.(type) {
+ case *ast.A_Const:
+ return n.Loc
+ default:
+ return ast.Loc{Start: -1, End: -1}
+ }
+}
diff --git a/backend/plugin/parser/pg/limit_test.go b/backend/plugin/parser/pg/limit_test.go
new file mode 100644
index 0000000000..43d5acfe4c
--- /dev/null
+++ b/backend/plugin/parser/pg/limit_test.go
@@ -0,0 +1,437 @@
+package pg
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestGetStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ want string
+ }{
+ {
+ name: "Simple SELECT without LIMIT",
+ statement: "SELECT * FROM users",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "SELECT with WHERE clause",
+ statement: "SELECT id, name FROM users WHERE active = true",
+ limit: 5,
+ want: "SELECT id, name FROM users WHERE active = true LIMIT 5",
+ },
+ {
+ name: "SELECT with existing LIMIT higher than requested",
+ statement: "SELECT * FROM users LIMIT 100",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "SELECT with existing LIMIT lower than requested",
+ statement: "SELECT * FROM users LIMIT 5",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 5",
+ },
+ {
+ name: "SELECT with existing LIMIT 0 is kept, not replaced",
+ statement: "SELECT * FROM users LIMIT 0",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 0",
+ },
+ {
+ name: "WITH query (CTE)",
+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",
+ limit: 10,
+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",
+ },
+ {
+ name: "SELECT with ORDER BY",
+ statement: "SELECT * FROM users ORDER BY created_at DESC",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
+ },
+ {
+ name: "SELECT with GROUP BY and HAVING",
+ statement: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5",
+ limit: 10,
+ want: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5 LIMIT 10",
+ },
+ {
+ name: "UNION query",
+ statement: "SELECT id FROM users UNION SELECT id FROM admins",
+ limit: 10,
+ want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",
+ },
+ {
+ name: "SELECT with OFFSET",
+ statement: "SELECT * FROM users LIMIT 20 OFFSET 10",
+ limit: 5,
+ want: "SELECT * FROM users LIMIT 5 OFFSET 10",
+ },
+ {
+ name: "SELECT with FOR UPDATE",
+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
+ },
+ {
+ name: "Non-SELECT statement (INSERT)",
+ statement: "INSERT INTO users (name) VALUES ('John')",
+ limit: 10,
+ want: "INSERT INTO users (name) VALUES ('John')",
+ },
+ {
+ name: "Non-SELECT statement (UPDATE)",
+ statement: "UPDATE users SET active = false WHERE id = 1",
+ limit: 10,
+ want: "UPDATE users SET active = false WHERE id = 1",
+ },
+ {
+ name: "Non-SELECT statement (DELETE)",
+ statement: "DELETE FROM users WHERE active = false",
+ limit: 10,
+ want: "DELETE FROM users WHERE active = false",
+ },
+ {
+ name: "SELECT with comments",
+ statement: "SELECT * FROM users -- This is a comment\nWHERE active = true",
+ limit: 10,
+ want: "SELECT * FROM users -- This is a comment\nWHERE active = true LIMIT 10",
+ },
+ {
+ name: "Complex nested SELECT",
+ statement: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01'",
+ limit: 10,
+ want: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01' LIMIT 10",
+ },
+ {
+ name: "SELECT with JOIN",
+ statement: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id",
+ limit: 10,
+ want: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id LIMIT 10",
+ },
+ {
+ name: "SELECT with parentheses",
+ statement: "(SELECT * FROM users)",
+ limit: 10,
+ want: "(SELECT * FROM users) LIMIT 10",
+ },
+ {
+ name: "SELECT with nested parentheses",
+ statement: "((SELECT * FROM users))",
+ limit: 10,
+ want: "((SELECT * FROM users)) LIMIT 10",
+ },
+ {
+ name: "SELECT with parentheses and existing LIMIT higher than requested",
+ statement: "(SELECT * FROM users LIMIT 100)",
+ limit: 10,
+ want: "(SELECT * FROM users LIMIT 10)",
+ },
+ {
+ name: "SELECT with parentheses and existing LIMIT lower than requested",
+ statement: "(SELECT * FROM users LIMIT 5)",
+ limit: 10,
+ want: "(SELECT * FROM users LIMIT 5)",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := statementWithResultLimit(tt.statement, tt.limit, "")
+
+ // For non-SELECT statements, the parser might fail and fall back to CTE approach
+ // which we should avoid for non-SELECT statements
+ if tt.statement == tt.want {
+ // Non-SELECT statements should remain unchanged
+ assert.Equal(t, tt.want, got)
+ } else {
+ // For SELECT statements, check if LIMIT was properly added
+ // The exact format might vary slightly due to parser normalization
+ assert.Contains(t, got, "LIMIT")
+ assert.Contains(t, got, fmt.Sprintf("%d", tt.limit))
+ }
+ })
+ }
+}
+
+func TestGetStatementWithResultLimitInline(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ wantErr bool
+ want string
+ }{
+ {
+ name: "Valid SELECT statement",
+ statement: "SELECT * FROM users",
+ limit: 10,
+ wantErr: false,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "Invalid SQL syntax",
+ statement: "SELECT * FROM WHERE",
+ limit: 10,
+ wantErr: true,
+ },
+ {
+ name: "Empty statement",
+ statement: "",
+ limit: 10,
+ wantErr: true,
+ },
+ {
+ name: "FOR UPDATE gets proper whitespace",
+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
+ },
+ {
+ name: "FOR SHARE gets proper whitespace",
+ statement: "SELECT * FROM users FOR SHARE",
+ limit: 5,
+ want: "SELECT * FROM users LIMIT 5 FOR SHARE",
+ },
+ {
+ name: "SELECT ending with function call",
+ statement: "SELECT now()",
+ limit: 10,
+ want: "SELECT now() LIMIT 10",
+ },
+ {
+ name: "SELECT with subquery in WHERE",
+ statement: "SELECT * FROM users WHERE id IN (SELECT id FROM admins)",
+ limit: 10,
+ want: "SELECT * FROM users WHERE id IN (SELECT id FROM admins) LIMIT 10",
+ },
+ {
+ name: "Non-constant LIMIT expression falls back to error",
+ statement: "SELECT * FROM t LIMIT (1+2)",
+ limit: 5,
+ wantErr: true,
+ },
+ {
+ name: "LIMIT 0 is kept, not replaced with the requested cap",
+ statement: "SELECT * FROM t LIMIT 0",
+ limit: 5,
+ want: "SELECT * FROM t LIMIT 0",
+ },
+ {
+ name: "CTE with lateral and ORDER BY",
+ statement: `WITH params AS (
+ SELECT 'resource.environment_id in []'::text AS env_condition
+), affected AS (
+ SELECT
+ p.resource,
+ binding,
+ COALESCE(binding->'condition'->>'expression', '') AS old_expression
+ FROM policy p,
+ LATERAL jsonb_array_elements(p.payload->'bindings') AS binding
+ WHERE p.type = 'IAM'
+ AND p.resource_type = 'PROJECT'
+ AND binding->>'role' = 'roles/projectOwner'
+ AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'
+)
+SELECT
+ resource,
+ binding->'members' AS members,
+ binding->'condition' AS old_condition,
+ CASE
+ WHEN old_expression = '' THEN params.env_condition
+ ELSE '(' || old_expression || ') && ' || params.env_condition
+ END AS new_expression
+FROM affected
+CROSS JOIN params
+ORDER BY resource`,
+ limit: 1000,
+ want: `WITH params AS (
+ SELECT 'resource.environment_id in []'::text AS env_condition
+), affected AS (
+ SELECT
+ p.resource,
+ binding,
+ COALESCE(binding->'condition'->>'expression', '') AS old_expression
+ FROM policy p,
+ LATERAL jsonb_array_elements(p.payload->'bindings') AS binding
+ WHERE p.type = 'IAM'
+ AND p.resource_type = 'PROJECT'
+ AND binding->>'role' = 'roles/projectOwner'
+ AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'
+)
+SELECT
+ resource,
+ binding->'members' AS members,
+ binding->'condition' AS old_condition,
+ CASE
+ WHEN old_expression = '' THEN params.env_condition
+ ELSE '(' || old_expression || ') && ' || params.env_condition
+ END AS new_expression
+FROM affected
+CROSS JOIN params
+ORDER BY resource LIMIT 1000`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)
+ if tt.wantErr {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ assert.NotEmpty(t, got)
+ if tt.want != "" {
+ assert.Equal(t, tt.want, got)
+ }
+ }
+ })
+ }
+}
+
+func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ want string
+ clausesInOrder []string
+ }{
+ {
+ name: "ORDER BY",
+ statement: "SELECT * FROM users ORDER BY created_at DESC",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
+ clausesInOrder: []string{"ORDER BY", "LIMIT 10"},
+ },
+ {
+ name: "OFFSET without LIMIT",
+ statement: "SELECT * FROM users OFFSET 20",
+ limit: 10,
+ clausesInOrder: []string{"OFFSET 20", "LIMIT 10"},
+ },
+ {
+ name: "ORDER BY with OFFSET without LIMIT",
+ statement: "SELECT * FROM users ORDER BY created_at DESC OFFSET 20",
+ limit: 10,
+ clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10"},
+ },
+ {
+ name: "ORDER BY with FOR UPDATE",
+ statement: "SELECT * FROM users ORDER BY id FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",
+ clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},
+ },
+ {
+ name: "ORDER BY with OFFSET and FOR UPDATE",
+ statement: "SELECT * FROM users ORDER BY id OFFSET 20 FOR UPDATE",
+ limit: 10,
+ clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10", "FOR UPDATE"},
+ },
+ {
+ name: "GROUP BY HAVING WINDOW ORDER BY",
+ statement: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC",
+ limit: 10,
+ want: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC LIMIT 10",
+ clausesInOrder: []string{"GROUP BY", "HAVING", "WINDOW", "ORDER BY n", "LIMIT 10"},
+ },
+ {
+ name: "UNION with outer ORDER BY",
+ statement: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id",
+ limit: 10,
+ want: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id LIMIT 10",
+ clausesInOrder: []string{"UNION ALL", "ORDER BY", "LIMIT 10"},
+ },
+ {
+ name: "ORDER BY inside window only",
+ statement: "SELECT row_number() OVER (ORDER BY created_at) FROM users",
+ limit: 10,
+ want: "SELECT row_number() OVER (ORDER BY created_at) FROM users LIMIT 10",
+ clausesInOrder: []string{"FROM users", "LIMIT 10"},
+ },
+ {
+ name: "ORDER BY inside subquery only",
+ statement: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users",
+ limit: 10,
+ want: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users LIMIT 10",
+ clausesInOrder: []string{"AS ordered_users", "LIMIT 10"},
+ },
+ {
+ name: "WITH query with outer ORDER BY",
+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",
+ limit: 10,
+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",
+ clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)
+ require.NoError(t, err)
+ if tt.want != "" {
+ assert.Equal(t, tt.want, got)
+ }
+ assertSubstringsInOrder(t, got, tt.clausesInOrder)
+
+ _, err = ParsePg(got)
+ require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)
+ })
+ }
+}
+
+func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {
+ t.Helper()
+
+ start := 0
+ for _, substring := range substrings {
+ index := strings.Index(s[start:], substring)
+ require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)
+ start += index + len(substring)
+ }
+}
+
+// TestCockroachDBReusesPGLimitRewrite locks the fix that gave CockroachDB the
+// same precise rewrite PostgreSQL gets, through the shared registration in
+// limit.go's init(). CockroachDB used to blind-wrap every statement in a CTE
+// regardless of shape — a plain write like DELETE FROM t became
+// WITH result AS (DELETE FROM t) SELECT * FROM result LIMIT n, a statement the
+// read-only classifier could misread as a query with a data-modifying CTE. The
+// registered function must be the exact same one PostgreSQL uses, so a
+// non-SELECT statement passes through unchanged rather than being wrapped, and
+// a SELECT gets the same precise cap.
+func TestCockroachDBReusesPGLimitRewrite(t *testing.T) {
+ require.Equal(t, "DELETE FROM t",
+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "DELETE FROM t", 10, ""))
+ require.Equal(t, "SELECT * FROM t LIMIT 10",
+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "SELECT * FROM t LIMIT 100", 10, ""))
+ // The blind CTE-wrap CockroachDB used before this reuse preserved a literal
+ // LIMIT 0 (the inner zero always won regardless of the outer cap); the
+ // precise rewrite must keep preserving it rather than replacing it.
+ require.Equal(t, "SELECT * FROM t LIMIT 0",
+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "SELECT * FROM t LIMIT 0", 10, ""))
+}
+
+func TestUnlimitedResultLimit(t *testing.T) {
+ for _, clause := range []string{"ALL", "NULL"} {
+ t.Run(clause, func(t *testing.T) {
+ statement := "SELECT * FROM t LIMIT " + clause + " OFFSET 2;"
+ got := statementWithResultLimit(statement, 10, "")
+ require.Equal(t, got, base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, statement, 10, ""))
+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT "+clause+" OFFSET 2\n) SELECT * FROM result LIMIT 10;", got)
+ })
+ }
+}
diff --git a/backend/plugin/parser/plsql/limit.go b/backend/plugin/parser/plsql/limit.go
new file mode 100644
index 0000000000..01061ec6c0
--- /dev/null
+++ b/backend/plugin/parser/plsql/limit.go
@@ -0,0 +1,308 @@
+package plsql
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+ "unicode"
+
+ oracleast "github.com/bytebase/omni/oracle/ast"
+ oracleparser "github.com/bytebase/omni/oracle/parser"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/utils"
+)
+
+const dbVersion12 = 12
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_ORACLE, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for Oracle.
+// engineVersion picks the clause: ROWNUM before version 12, FETCH NEXT from
+// 12c on — see isOracle11gOrEarlier.
+func statementWithResultLimit(statement string, limit int, engineVersion string) string {
+ return addResultLimit(statement, limit, engineVersion)
+}
+
+// addResultLimit adds a limit clause to the statement based on Oracle version
+func addResultLimit(stmt string, limit int, engineVersion string) string {
+ // Check if we should skip adding limit (e.g., for simple DUAL queries)
+ if shouldSkipLimit(stmt) {
+ return stmt
+ }
+
+ // Determine Oracle version
+ if isOracle11gOrEarlier(engineVersion) {
+ return addLimitFor11g(stmt, limit)
+ }
+ return addLimitFor12cAndLater(stmt, limit)
+}
+
+// shouldSkipLimit checks if the statement needs a limit clause
+func shouldSkipLimit(stmt string) bool {
+ ok, err := skipAddLimit(stmt)
+ return err == nil && ok
+}
+
+// isOracle11gOrEarlier checks if the Oracle version is 11g or earlier
+func isOracle11gOrEarlier(engineVersion string) bool {
+ versionIdx := strings.Index(engineVersion, ".")
+ if versionIdx < 0 {
+ return true // Default to 11g behavior for invalid version
+ }
+ versionNumber, err := strconv.Atoi(engineVersion[:versionIdx])
+ if err != nil {
+ return true // Default to 11g behavior for parsing errors
+ }
+ return versionNumber < dbVersion12
+}
+
+// addLimitFor11g adds a ROWNUM-based limit for Oracle 11g and earlier versions.
+// Uses the legacy approach with subquery and ROWNUM.
+func addLimitFor11g(statement string, limitCount int) string {
+ if !isSelectOrWithStatement(statement) {
+ return statement
+ }
+ return fmt.Sprintf("SELECT * FROM (%s) WHERE ROWNUM <= %d", base.TrimStatement(statement), limitCount)
+}
+
+// isSelectOrWithStatement checks if the statement is a SELECT or WITH statement
+func isSelectOrWithStatement(statement string) bool {
+ trimmedStatement := strings.ToLower(strings.TrimLeftFunc(statement, unicode.IsSpace))
+ return strings.HasPrefix(trimmedStatement, "select") || strings.HasPrefix(trimmedStatement, "with")
+}
+
+// addLimitFor12cAndLater adds a FETCH NEXT clause for Oracle 12c and later versions.
+// Uses the modern SQL standard approach, falling back to 11g approach on error.
+func addLimitFor12cAndLater(statement string, limit int) string {
+ if !isSelectOrWithStatement(statement) {
+ return statement
+ }
+
+ stmt, err := addFetchNextClause(statement, limit)
+ if err != nil {
+ slog.Error("failed to add FETCH NEXT clause, falling back to ROWNUM",
+ slog.String("statement", statement), log.BBError(err))
+ return addLimitFor11g(statement, limit)
+ }
+ return stmt
+}
+
+// addFetchNextClause adds a FETCH NEXT clause to a SELECT statement using AST parsing.
+// This provides more precise placement of the limit clause compared to simple string wrapping.
+func addFetchNextClause(statement string, limitCount int) (string, error) {
+ list, err := ParsePLSQL(statement)
+ if err != nil {
+ return "", err
+ }
+ if list == nil || len(list.Items) == 0 {
+ return "", errors.New("no parse results")
+ }
+ if len(list.Items) > 1 {
+ return "", errors.Errorf("expected single statement, got %d statements", len(list.Items))
+ }
+ raw, ok := list.Items[0].(*oracleast.RawStmt)
+ if !ok {
+ return "", errors.Errorf("expected raw statement, got %T", list.Items[0])
+ }
+ selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)
+ if !ok {
+ return statement, nil
+ }
+
+ res, err := rewriteOracleSelectFetch(statement, selectStmt, limitCount)
+ if err != nil {
+ return "", err
+ }
+ // https://stackoverflow.com/questions/27987882/how-can-i-solve-ora-00911-invalid-character-error
+ res = strings.TrimRightFunc(res, utils.IsSpaceOrSemicolon)
+
+ return res, nil
+}
+
+func rewriteOracleSelectFetch(sql string, selectStmt *oracleast.SelectStmt, limitCount int) (string, error) {
+ target := rightmostOracleSetSelect(selectStmt)
+ if target.FetchFirst != nil {
+ return rewriteOracleFetchClause(sql, target.FetchFirst, limitCount)
+ }
+ if target.ForUpdate != nil && target.ForUpdate.Loc.Start > 0 && target.ForUpdate.Loc.Start <= len(sql) {
+ return sql[:target.ForUpdate.Loc.Start] + fmt.Sprintf("FETCH NEXT %d ROWS ONLY ", limitCount) + sql[target.ForUpdate.Loc.Start:], nil
+ }
+
+ loc := oracleast.NodeLoc(target)
+ if loc.End < 0 || loc.End > len(sql) {
+ return "", errors.Errorf("invalid SELECT end position %d", loc.End)
+ }
+ return sql[:loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[loc.End:], nil
+}
+
+func rightmostOracleSetSelect(selectStmt *oracleast.SelectStmt) *oracleast.SelectStmt {
+ if selectStmt.Op != oracleast.SETOP_NONE && selectStmt.Rarg != nil {
+ return rightmostOracleSetSelect(selectStmt.Rarg)
+ }
+ return selectStmt
+}
+
+func rewriteOracleFetchClause(sql string, fetch *oracleast.FetchFirstClause, limitCount int) (string, error) {
+ if fetch.Count == nil {
+ if fetch.Loc.Start < 0 || fetch.Loc.End < fetch.Loc.Start || fetch.Loc.End > len(sql) {
+ return "", errors.Errorf("invalid FETCH position %d:%d", fetch.Loc.Start, fetch.Loc.End)
+ }
+ if hasOracleFetchKeyword(sql, fetch.Loc) {
+ return sql, nil
+ }
+ return sql[:fetch.Loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[fetch.Loc.End:], nil
+ }
+ if fetch.Percent {
+ return "", errors.Errorf("cannot rewrite PERCENT FETCH expression")
+ }
+
+ existingLimit := extractOracleFetchCount(fetch.Count)
+ if existingLimit > 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+
+ loc := oracleast.NodeLoc(fetch.Count)
+ loc = trimOracleLocSpace(sql, loc)
+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
+ if existingLimit <= 0 {
+ return "", errors.Errorf("cannot rewrite non-constant FETCH expression")
+ }
+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
+ }
+ return "", errors.Errorf("cannot rewrite FETCH expression")
+}
+
+func hasOracleFetchKeyword(sql string, loc oracleast.Loc) bool {
+ segment := sql[loc.Start:loc.End]
+ lexer := oracleparser.NewLexer(segment)
+ for {
+ tok := lexer.NextToken()
+ if tok.Loc == tok.End && tok.End >= len(segment) {
+ return false
+ }
+ if tok.Loc < 0 || tok.End > len(segment) || tok.End <= tok.Loc {
+ return false
+ }
+ if strings.EqualFold(segment[tok.Loc:tok.End], "FETCH") {
+ return true
+ }
+ }
+}
+
+func extractOracleFetchCount(node oracleast.Node) int {
+ limit, ok := node.(*oracleast.NumberLiteral)
+ if !ok || limit.IsFloat || limit.Ival <= 0 {
+ return 0
+ }
+ return int(limit.Ival)
+}
+
+func trimOracleLocSpace(sql string, loc oracleast.Loc) oracleast.Loc {
+ for loc.Start < loc.End && loc.Start < len(sql) && unicode.IsSpace(rune(sql[loc.Start])) {
+ loc.Start++
+ }
+ for loc.End > loc.Start && loc.End <= len(sql) && unicode.IsSpace(rune(sql[loc.End-1])) {
+ loc.End--
+ }
+ return loc
+}
+
+// skipAddLimit checks if the statement needs a limit clause.
+// For Oracle, we think the statement like "SELECT xxx FROM DUAL" does not need a limit clause.
+// More details, xxx can not be a subquery.
+func skipAddLimit(stmt string) (bool, error) {
+ list, err := ParsePLSQL(stmt)
+ if err != nil {
+ return false, err
+ }
+ if list == nil || len(list.Items) == 0 {
+ return false, nil
+ }
+ // Multiple statements should not skip limit
+ if len(list.Items) > 1 {
+ return false, nil
+ }
+ raw, ok := list.Items[0].(*oracleast.RawStmt)
+ if !ok {
+ return false, nil
+ }
+ selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)
+ if !ok {
+ return false, nil
+ }
+ if !isSimpleOracleSelect(selectStmt) {
+ return false, nil
+ }
+ if !isOracleSelectFromDual(selectStmt) {
+ return false, nil
+ }
+ return !hasOracleSubqueriesInSelection(selectStmt), nil
+}
+
+func isSimpleOracleSelect(selectStmt *oracleast.SelectStmt) bool {
+ return selectStmt.WithClause == nil &&
+ !selectStmt.Distinct &&
+ !selectStmt.UniqueKw &&
+ !selectStmt.All &&
+ selectStmt.Into == nil &&
+ selectStmt.IntoVars == nil &&
+ selectStmt.WhereClause == nil &&
+ selectStmt.Hierarchical == nil &&
+ selectStmt.GroupClause == nil &&
+ selectStmt.HavingClause == nil &&
+ selectStmt.ModelClause == nil &&
+ len(selectStmt.WindowDefs) == 0 &&
+ selectStmt.QualifyClause == nil &&
+ selectStmt.OrderBy == nil &&
+ selectStmt.ForUpdate == nil &&
+ selectStmt.FetchFirst == nil &&
+ selectStmt.Op == oracleast.SETOP_NONE
+}
+
+func isOracleSelectFromDual(selectStmt *oracleast.SelectStmt) bool {
+ if selectStmt.FromClause == nil || len(selectStmt.FromClause.Items) != 1 {
+ return false
+ }
+ tableRef, ok := selectStmt.FromClause.Items[0].(*oracleast.TableRef)
+ if !ok || tableRef.Name == nil {
+ return false
+ }
+ return tableRef.Alias == nil &&
+ tableRef.Name.Schema == "" &&
+ tableRef.Name.DBLink == "" &&
+ strings.EqualFold(tableRef.Name.Name, "DUAL")
+}
+
+func hasOracleSubqueriesInSelection(selectStmt *oracleast.SelectStmt) bool {
+ if selectStmt.TargetList == nil || len(selectStmt.TargetList.Items) == 0 {
+ return true
+ }
+ for _, item := range selectStmt.TargetList.Items {
+ target, ok := item.(*oracleast.ResTarget)
+ if !ok || target.Expr == nil {
+ return true
+ }
+ if _, ok := target.Expr.(*oracleast.Star); ok {
+ return true
+ }
+ hasSubquery := false
+ oracleast.Inspect(target.Expr, func(node oracleast.Node) bool {
+ if _, ok := node.(*oracleast.SubqueryExpr); ok {
+ hasSubquery = true
+ return false
+ }
+ return true
+ })
+ if hasSubquery {
+ return true
+ }
+ }
+ return false
+}
diff --git a/backend/plugin/db/oracle/query_test.go b/backend/plugin/parser/plsql/limit_test.go
similarity index 97%
rename from backend/plugin/db/oracle/query_test.go
rename to backend/plugin/parser/plsql/limit_test.go
index c3bd7ccfe1..7d327dec36 100644
--- a/backend/plugin/db/oracle/query_test.go
+++ b/backend/plugin/parser/plsql/limit_test.go
@@ -1,4 +1,4 @@
-package oracle
+package plsql
import (
"io"
@@ -11,7 +11,6 @@ import (
"gopkg.in/yaml.v3"
"github.com/bytebase/bytebase/backend/common/yamltest"
- plsqlparser "github.com/bytebase/bytebase/backend/plugin/parser/plsql"
)
func TestGetStatementWithResultLimit(t *testing.T) {
@@ -51,7 +50,7 @@ func TestAddResultLimitKeepsNonSelectStatementsUnchanged(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- got := addResultLimit(tc.statement, 10, tc.engineVersion)
+ got := statementWithResultLimit(tc.statement, 10, tc.engineVersion)
require.Equal(t, tc.statement, got)
})
@@ -83,7 +82,7 @@ func TestAddResultLimitSkipsOnlySimpleDualSelect(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- got := addResultLimit(tc.statement, 10, "19.0.0.0.0")
+ got := statementWithResultLimit(tc.statement, 10, "19.0.0.0.0")
require.Equal(t, tc.want, got)
})
@@ -157,7 +156,7 @@ func TestAddLimitFor12cAndLaterRegressionClauseOrder(t *testing.T) {
require.Equal(t, tc.want, got)
assertSubstringsInOrder(t, got, tc.clausesInOrder)
if tc.validateParse {
- _, err := plsqlparser.ParsePLSQL(got)
+ _, err := ParsePLSQL(got)
require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)
}
})
diff --git a/backend/plugin/db/oracle/test-data/test_limit.yaml b/backend/plugin/parser/plsql/test-data/test_limit.yaml
similarity index 100%
rename from backend/plugin/db/oracle/test-data/test_limit.yaml
rename to backend/plugin/parser/plsql/test-data/test_limit.yaml
diff --git a/backend/plugin/parser/redshift/limit.go b/backend/plugin/parser/redshift/limit.go
new file mode 100644
index 0000000000..15290db615
--- /dev/null
+++ b/backend/plugin/parser/redshift/limit.go
@@ -0,0 +1,134 @@
+package redshift
+
+import (
+ "fmt"
+ "log/slog"
+ "strings"
+
+ redshiftast "github.com/bytebase/omni/redshift/ast"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_REDSHIFT, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for Redshift, which
+// takes no engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := statementWithResultLimitInline(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ // To handle cases where there are comments in the query.
+ // eg. select * from t1 -- this is comment;
+ // Add two new line symbols here.
+ return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
+
+func statementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ stmts, err := ParseRedshift(statement)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to parse statement")
+ }
+ if len(stmts) != 1 {
+ return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))
+ }
+
+ sel, ok := stmts[0].AST.(*redshiftast.SelectStmt)
+ if !ok {
+ // Non-SELECT statement, return as-is.
+ return statement, nil
+ }
+
+ return rewriteSelectLimit(statement, sel, limitCount)
+}
+
+// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement
+// using byte-offset positions from the omni AST to surgically edit the original SQL text.
+func rewriteSelectLimit(sql string, sel *redshiftast.SelectStmt, limitCount int) (string, error) {
+ if sel.LimitCount != nil {
+ existingLimit, isInteger := extractIntFromNode(sel.LimitCount)
+ loc := nodeLocOf(sel.LimitCount)
+ if isInteger && existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil // existing limit is already lower or equal, keep it
+ }
+ if isInteger && loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {
+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil
+ }
+ // ALL/NULL and non-constant limits lack reliable replacement spans.
+ // Cap them with the outer query instead of rewriting their text.
+ return "", errors.Errorf("cannot rewrite LIMIT expression in place")
+ }
+
+ // No LIMIT clause — find the right insertion point.
+ // Redshift's PostgreSQL-derived grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...
+ // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.
+ insertPos, beforeLocking := findLimitInsertPosition(sel)
+ if beforeLocking {
+ // Inserting at the start of FOR UPDATE/SHARE. The original whitespace
+ // before FOR becomes the separator before LIMIT; we add a trailing
+ // space to separate the limit value from FOR.
+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil
+ }
+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil
+}
+
+// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,
+// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).
+func findLimitInsertPosition(sel *redshiftast.SelectStmt) (int, bool) {
+ // LIMIT must appear before FOR UPDATE/SHARE.
+ if sel.LockingClause != nil {
+ span := redshiftast.ListSpan(sel.LockingClause)
+ if span.Start > 0 {
+ return span.Start, true
+ }
+ }
+
+ // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).
+ // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,
+ // so account for the sort clause explicitly.
+ end := sel.Loc.End
+ if span := redshiftast.ListSpan(sel.SortClause); span.End > end {
+ end = span.End
+ }
+ if end <= 0 {
+ return 0, false
+ }
+ return end, false
+}
+
+// extractIntFromNode distinguishes integer limits, including zero, from unlimited
+// ALL/NULL constants and expressions that require the fallback wrapper.
+func extractIntFromNode(node redshiftast.Node) (int, bool) {
+ switch n := node.(type) {
+ case *redshiftast.Integer:
+ return int(n.Ival), true
+ case *redshiftast.A_Const:
+ if iv, ok := n.Val.(*redshiftast.Integer); ok {
+ return int(iv.Ival), true
+ }
+ return 0, false
+ default:
+ return 0, false
+ }
+}
+
+// nodeLocOf returns the Loc of a node, handling common wrapper types.
+func nodeLocOf(node redshiftast.Node) redshiftast.Loc {
+ switch n := node.(type) {
+ case *redshiftast.A_Const:
+ return n.Loc
+ default:
+ return redshiftast.Loc{Start: -1, End: -1}
+ }
+}
diff --git a/backend/plugin/parser/redshift/limit_test.go b/backend/plugin/parser/redshift/limit_test.go
new file mode 100644
index 0000000000..ae2eabb7c9
--- /dev/null
+++ b/backend/plugin/parser/redshift/limit_test.go
@@ -0,0 +1,228 @@
+package redshift
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ want string
+ }{
+ {
+ name: "Simple SELECT without LIMIT",
+ statement: "SELECT * FROM users",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "SELECT with existing LIMIT higher than requested",
+ statement: "SELECT * FROM users LIMIT 100",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "SELECT with existing LIMIT lower than requested",
+ statement: "SELECT * FROM users LIMIT 5",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 5",
+ },
+ {
+ name: "SELECT with existing LIMIT 0 is kept, not replaced",
+ statement: "SELECT * FROM users LIMIT 0",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 0",
+ },
+ {
+ name: "WITH query (CTE)",
+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",
+ limit: 10,
+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",
+ },
+ {
+ name: "SELECT with ORDER BY",
+ statement: "SELECT * FROM users ORDER BY created_at DESC",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
+ },
+ {
+ name: "UNION query",
+ statement: "SELECT id FROM users UNION SELECT id FROM admins",
+ limit: 10,
+ want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",
+ },
+ {
+ name: "SELECT with FOR UPDATE",
+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
+ },
+ {
+ name: "Non-SELECT statement (INSERT)",
+ statement: "INSERT INTO users (name) VALUES ('John')",
+ limit: 10,
+ want: "INSERT INTO users (name) VALUES ('John')",
+ },
+ {
+ name: "Non-SELECT statement (DELETE)",
+ statement: "DELETE FROM users WHERE active = false",
+ limit: 10,
+ want: "DELETE FROM users WHERE active = false",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := statementWithResultLimit(tt.statement, tt.limit, "")
+
+ if tt.statement == tt.want {
+ assert.Equal(t, tt.want, got)
+ } else {
+ assert.Contains(t, got, "LIMIT")
+ }
+ })
+ }
+}
+
+func TestGetStatementWithResultLimitInline(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ wantErr bool
+ want string
+ }{
+ {
+ name: "Valid SELECT statement",
+ statement: "SELECT * FROM users",
+ limit: 10,
+ want: "SELECT * FROM users LIMIT 10",
+ },
+ {
+ name: "Invalid SQL syntax",
+ statement: "SELECT * FROM WHERE",
+ limit: 10,
+ wantErr: true,
+ },
+ {
+ name: "Empty statement",
+ statement: "",
+ limit: 10,
+ wantErr: true,
+ },
+ {
+ name: "FOR UPDATE gets proper whitespace",
+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",
+ },
+ {
+ name: "Non-constant LIMIT expression falls back to error",
+ statement: "SELECT * FROM t LIMIT (1+2)",
+ limit: 5,
+ wantErr: true,
+ },
+ {
+ name: "LIMIT 0 is kept, not replaced with the requested cap",
+ statement: "SELECT * FROM t LIMIT 0",
+ limit: 5,
+ want: "SELECT * FROM t LIMIT 0",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)
+ if tt.wantErr {
+ require.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.NotEmpty(t, got)
+ if tt.want != "" {
+ assert.Equal(t, tt.want, got)
+ }
+ })
+ }
+}
+
+// TestGetStatementWithResultLimitFallback locks Redshift's own fallback text,
+// distinct from PostgreSQL's: a blind CTE-wrap for a statement the AST rewrite
+// cannot handle, still never applied to a statement that already parsed clean.
+func TestGetStatementWithResultLimitFallback(t *testing.T) {
+ got := statementWithResultLimit("SELECT * FROM t LIMIT (1+2)", 5, "")
+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT (1+2)\n) SELECT * FROM result LIMIT 5;", got)
+}
+
+func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ limit int
+ want string
+ clausesInOrder []string
+ }{
+ {
+ name: "ORDER BY",
+ statement: "SELECT * FROM users ORDER BY created_at DESC",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",
+ clausesInOrder: []string{"ORDER BY", "LIMIT 10"},
+ },
+ {
+ name: "ORDER BY with FOR UPDATE",
+ statement: "SELECT * FROM users ORDER BY id FOR UPDATE",
+ limit: 10,
+ want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",
+ clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},
+ },
+ {
+ name: "WITH query with outer ORDER BY",
+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",
+ limit: 10,
+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",
+ clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)
+ require.NoError(t, err)
+ if tt.want != "" {
+ assert.Equal(t, tt.want, got)
+ }
+ assertSubstringsInOrder(t, got, tt.clausesInOrder)
+
+ _, err = ParseRedshift(got)
+ require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)
+ })
+ }
+}
+
+func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {
+ t.Helper()
+
+ start := 0
+ for _, substring := range substrings {
+ index := strings.Index(s[start:], substring)
+ require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)
+ start += index + len(substring)
+ }
+}
+
+func TestUnlimitedResultLimit(t *testing.T) {
+ for _, clause := range []string{"ALL", "NULL"} {
+ t.Run(clause, func(t *testing.T) {
+ statement := "SELECT * FROM t LIMIT " + clause + " OFFSET 2;"
+ got := statementWithResultLimit(statement, 10, "")
+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT "+clause+" OFFSET 2\n) SELECT * FROM result LIMIT 10;", got)
+ })
+ }
+}
diff --git a/backend/plugin/db/snowflake/query.go b/backend/plugin/parser/snowflake/limit.go
similarity index 83%
rename from backend/plugin/db/snowflake/query.go
rename to backend/plugin/parser/snowflake/limit.go
index 2d32289c93..609ae35d2d 100644
--- a/backend/plugin/db/snowflake/query.go
+++ b/backend/plugin/parser/snowflake/limit.go
@@ -11,22 +11,27 @@ import (
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common/log"
- "github.com/bytebase/bytebase/backend/plugin/db/util"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/utils"
)
-const stmtErrFmt = "statement: %s"
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_SNOWFLAKE, statementWithResultLimit)
+}
-func getStatementWithResultLimit(statement string, limit int) string {
- stmt, err := getStatementWithResultLimitInline(statement, limit)
+// statementWithResultLimit implements base.ResultLimitFunc for Snowflake, which
+// takes no engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := statementWithResultLimitInline(statement, limit)
if err != nil {
slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
- return fmt.Sprintf("SELECT * FROM (%s) LIMIT %d", util.TrimStatement(statement), limit)
+ return fmt.Sprintf("SELECT * FROM (%s) LIMIT %d", base.TrimStatement(statement), limit)
}
return stmt
}
-// getStatementWithResultLimitInline rewrites the single statement so that its
+// statementWithResultLimitInline rewrites the single statement so that its
// outermost query returns at most limitCount rows. The rewrite splices the
// replacement into the original statement text via omni AST byte offsets
// (ast.Loc), preserving the original formatting. The output shape mirrors the
@@ -44,12 +49,12 @@ func getStatementWithResultLimit(statement string, limit int) string {
// Non-query statements are returned unchanged (the legacy listener never
// fired on them). Any error makes the caller fall back to wrapping the
// statement in SELECT * FROM (...) LIMIT n.
-func getStatementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {
+func statementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {
trimmed := strings.TrimRightFunc(singleStatement, utils.IsSpaceOrSemicolon)
file, err := omniparser.Parse(trimmed)
if err != nil {
- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)
+ return "", errors.Wrapf(err, "statement: %s", singleStatement)
}
if len(file.Stmts) != 1 {
return "", errors.Errorf("expected exactly 1 statement, got %d", len(file.Stmts))
@@ -65,7 +70,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (
if sel.Limit != nil {
spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Limit), limitCount)
if err != nil {
- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)
+ return "", errors.Wrapf(err, "statement: %s", singleStatement)
}
return spliced + "\n;", nil
}
@@ -74,7 +79,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (
if sel.Fetch != nil {
spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Fetch.Count), limitCount)
if err != nil {
- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)
+ return "", errors.Wrapf(err, "statement: %s", singleStatement)
}
return spliced + "\n;", nil
}
@@ -90,7 +95,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (
if sel.Top != nil {
spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Top), limitCount)
if err != nil {
- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)
+ return "", errors.Wrapf(err, "statement: %s", singleStatement)
}
return spliced + "\n;", nil
}
diff --git a/backend/plugin/db/snowflake/query_test.go b/backend/plugin/parser/snowflake/limit_test.go
similarity index 94%
rename from backend/plugin/db/snowflake/query_test.go
rename to backend/plugin/parser/snowflake/limit_test.go
index 680c8ccee4..55ab8de248 100644
--- a/backend/plugin/db/snowflake/query_test.go
+++ b/backend/plugin/parser/snowflake/limit_test.go
@@ -35,7 +35,7 @@ func runLimitTest(t *testing.T, file string, record bool) {
require.NoError(t, err)
for i, tc := range testCases {
- want := getStatementWithResultLimit(tc.Stmt, tc.Limit)
+ want := statementWithResultLimit(tc.Stmt, tc.Limit, "")
if record {
testCases[i].Want = want
} else {
diff --git a/backend/plugin/db/snowflake/test-data/test_limit.yaml b/backend/plugin/parser/snowflake/test-data/test_limit.yaml
similarity index 100%
rename from backend/plugin/db/snowflake/test-data/test_limit.yaml
rename to backend/plugin/parser/snowflake/test-data/test_limit.yaml
diff --git a/backend/plugin/parser/spanner/limit.go b/backend/plugin/parser/spanner/limit.go
new file mode 100644
index 0000000000..7d15ba8172
--- /dev/null
+++ b/backend/plugin/parser/spanner/limit.go
@@ -0,0 +1,42 @@
+package spanner
+
+import (
+ "strconv"
+
+ "github.com/bytebase/omni/googlesql/ast"
+ "github.com/bytebase/omni/googlesql/parser"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_SPANNER, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for Spanner, which
+// takes no engineVersion. It sets the LIMIT of the outermost query in stmt to at
+// most limit. It never wraps stmt in a subquery, where Spanner rejects a WITH
+// clause, so a query it cannot rewrite runs unchanged and queryStatement caps its
+// rows.
+func statementWithResultLimit(stmt string, limit int, _ string) string {
+ file, errs := parser.Parse(stmt)
+ if len(errs) > 0 || len(file.Stmts) != 1 {
+ return stmt
+ }
+ query, ok := file.Stmts[0].(*ast.QueryStmt)
+ // FOR UPDATE fails in queryStatement's read-only transaction, so there is no
+ // point fitting a LIMIT before it.
+ if !ok || query.ForUpdate {
+ return stmt
+ }
+ if query.Limit == nil {
+ end := query.Loc.End
+ return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]
+ }
+ count, ok := query.Limit.(*ast.Literal)
+ if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {
+ return stmt
+ }
+ return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]
+}
diff --git a/backend/plugin/parser/spanner/limit_test.go b/backend/plugin/parser/spanner/limit_test.go
new file mode 100644
index 0000000000..6d15c84587
--- /dev/null
+++ b/backend/plugin/parser/spanner/limit_test.go
@@ -0,0 +1,81 @@
+package spanner
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want string
+ }{
+ {
+ name: "select",
+ statement: "SELECT * FROM t",
+ want: "SELECT * FROM t LIMIT 100",
+ },
+ {
+ name: "with",
+ statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",
+ want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",
+ },
+ {
+ name: "lowercase with, order by, and trailing semicolon",
+ statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",
+ want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",
+ },
+ {
+ name: "with and set operation",
+ statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",
+ want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",
+ },
+ {
+ name: "set operation",
+ statement: "SELECT 1 UNION ALL SELECT 2",
+ want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",
+ },
+ {
+ name: "inner limit belongs to the parenthesized query",
+ statement: "(SELECT x FROM t LIMIT 500)",
+ want: "(SELECT x FROM t LIMIT 500) LIMIT 100",
+ },
+ {
+ name: "trailing comment",
+ statement: "SELECT x FROM t -- note",
+ want: "SELECT x FROM t LIMIT 100 -- note",
+ },
+ {
+ name: "lower limit is kept",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
+ },
+ {
+ name: "higher limit is lowered",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",
+ },
+ {
+ name: "non-literal limit is unchanged",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
+ },
+ {
+ name: "for update is unchanged",
+ statement: "SELECT x FROM t ORDER BY x FOR UPDATE",
+ want: "SELECT x FROM t ORDER BY x FOR UPDATE",
+ },
+ {
+ name: "unparsable query is unchanged",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ require.Equal(t, tc.want, statementWithResultLimit(tc.statement, 100, ""))
+ })
+ }
+}
diff --git a/backend/plugin/parser/standard/limit.go b/backend/plugin/parser/standard/limit.go
new file mode 100644
index 0000000000..5de8c86391
--- /dev/null
+++ b/backend/plugin/parser/standard/limit.go
@@ -0,0 +1,20 @@
+package standard
+
+import (
+ "fmt"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_CLICKHOUSE, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for ClickHouse,
+// which takes no engineVersion. ClickHouse is parsed only as text here (see
+// query.go), so there is no AST to place the LIMIT precisely; every statement
+// is wrapped in a CTE.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)
+}
diff --git a/backend/plugin/parser/standard/limit_test.go b/backend/plugin/parser/standard/limit_test.go
new file mode 100644
index 0000000000..c5a7678a0b
--- /dev/null
+++ b/backend/plugin/parser/standard/limit_test.go
@@ -0,0 +1,23 @@
+package standard
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ statement string
+ limit int
+ want string
+ }{
+ {"SELECT * FROM t", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},
+ {"SELECT * FROM t;", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},
+ {"DELETE FROM t", 5, "WITH result AS (DELETE FROM t) SELECT * FROM result LIMIT 5;"},
+ }
+ for _, tc := range tests {
+ got := statementWithResultLimit(tc.statement, tc.limit, "")
+ require.Equalf(t, tc.want, got, "%q", tc.statement)
+ }
+}
diff --git a/backend/plugin/parser/starrocks/limit.go b/backend/plugin/parser/starrocks/limit.go
new file mode 100644
index 0000000000..80f3ac49f8
--- /dev/null
+++ b/backend/plugin/parser/starrocks/limit.go
@@ -0,0 +1,227 @@
+package starrocks
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ "github.com/bytebase/omni/starrocks/ast"
+ "github.com/bytebase/omni/starrocks/parser"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_STARROCKS, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for StarRocks, which
+// takes no engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ trimmedStatement := strings.TrimSpace(statement)
+ if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {
+ return statement
+ }
+
+ stmt, err := statementWithResultLimitInline(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
+
+func statementWithResultLimitInline(statement string, limitCount int) (string, error) {
+ if strings.TrimSpace(statement) == "" {
+ return "", errors.New("empty statement")
+ }
+
+ file, errs := parser.Parse(statement)
+ if len(errs) > 0 {
+ return "", errors.New(errs[0].Error())
+ }
+ if file == nil || len(file.Stmts) != 1 {
+ stmtCount := 0
+ if file != nil {
+ stmtCount = len(file.Stmts)
+ }
+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)
+ }
+
+ switch stmt := file.Stmts[0].(type) {
+ case *ast.SelectStmt:
+ return rewriteSelectLimit(statement, stmt, limitCount)
+ case *ast.SetOpStmt:
+ return rewriteSetOpLimit(statement, stmt, limitCount)
+ case *ast.ParenSelect:
+ // The wrapper itself may carry the trailing clauses —
+ // (SELECT 1) LIMIT 900000 attaches the LIMIT to the ParenSelect,
+ // not the inner query — so the wrapper is where the cap applies.
+ if stmt.Limit != nil {
+ return rewriteExistingLimit(statement, stmt.Limit, limitCount)
+ }
+ // No wrapper LIMIT: append one at the wrapper level, which covers
+ // any paren nesting depth — ((SELECT 1)) LIMIT n and
+ // (SELECT 1) ORDER BY 1 LIMIT n are engine-valid — instead of
+ // unwrapping and missing deeper layers.
+ if hasUnparsedTail(statement, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
+ }
+ return statement[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + statement[stmt.Loc.End:], nil
+ default:
+ return statement, nil
+ }
+}
+
+func hasUnparsedTail(sql string, locEnd int) bool {
+ pos := locEnd
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ for pos < len(sql) && sql[pos] == ';' {
+ pos++
+ }
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if pos >= len(sql) {
+ return false
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return false
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return false
+ }
+ pos += end + 2
+ continue
+ }
+ return true
+ }
+ return false
+}
+
+// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it
+// is a constant no larger than limitCount, replace the count otherwise.
+func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {
+ limitLoc := literalLoc(limitNode)
+ if limitLoc.Start < 0 {
+ return "", errors.New("cannot rewrite non-constant LIMIT expression")
+ }
+
+ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {
+ // A literal 0 count (LIMIT offset,0) is a valid, stricter limit and must
+ // be kept, matching the non-comma path below.
+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])
+ if existingCount >= 0 && existingCount <= limitCount {
+ return sql, nil
+ }
+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil
+ }
+
+ existingLimit := extractLimitValue(limitNode)
+ if existingLimit >= 0 && existingLimit <= limitCount {
+ return sql, nil
+ }
+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil
+}
+
+func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil {
+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)
+ }
+
+ if stmt.Into != nil {
+ return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil
+ }
+ if hasUnparsedTail(sql, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
+ }
+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
+}
+
+func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {
+ if stmt.Limit != nil {
+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)
+ }
+ if hasUnparsedTail(sql, stmt.Loc.End) {
+ return "", errors.New("statement has unparsed tail content")
+ }
+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil
+}
+
+func extractLimitValue(node ast.Node) int {
+ lit, ok := node.(*ast.Literal)
+ if !ok || lit.Kind != ast.LitInt {
+ return -1
+ }
+ val, err := strconv.Atoi(lit.Value)
+ if err != nil {
+ return -1
+ }
+ return val
+}
+
+func literalLoc(node ast.Node) ast.Loc {
+ lit, ok := node.(*ast.Literal)
+ if !ok {
+ return ast.Loc{Start: -1, End: -1}
+ }
+ return lit.Loc
+}
+
+func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {
+ pos := limitLoc.End
+ pos = skipWhitespaceAndComments(sql, pos)
+ if pos >= len(sql) || sql[pos] != ',' {
+ return 0, 0, false
+ }
+ pos++
+ pos = skipWhitespaceAndComments(sql, pos)
+ countStart = pos
+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {
+ pos++
+ }
+ if pos == countStart {
+ return 0, 0, false
+ }
+ return countStart, pos, true
+}
+
+func skipWhitespaceAndComments(sql string, pos int) int {
+ for pos < len(sql) {
+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {
+ pos++
+ }
+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {
+ nl := strings.IndexByte(sql[pos:], '\n')
+ if nl < 0 {
+ return len(sql)
+ }
+ pos += nl + 1
+ continue
+ }
+ if strings.HasPrefix(sql[pos:], "/*") {
+ end := strings.Index(sql[pos:], "*/")
+ if end < 0 {
+ return len(sql)
+ }
+ pos += end + 2
+ continue
+ }
+ return pos
+ }
+ return pos
+}
diff --git a/backend/plugin/db/starrocks/query_test.go b/backend/plugin/parser/starrocks/limit_test.go
similarity index 97%
rename from backend/plugin/db/starrocks/query_test.go
rename to backend/plugin/parser/starrocks/limit_test.go
index c50d65dda4..6ffec593af 100644
--- a/backend/plugin/db/starrocks/query_test.go
+++ b/backend/plugin/parser/starrocks/limit_test.go
@@ -94,6 +94,12 @@ func TestGetStatementWithResultLimit(t *testing.T) {
count: 10,
want: "SELECT * FROM t LIMIT 123,10;",
},
+ {
+ // A literal 0 count is kept, not replaced with the requested cap.
+ stmt: "SELECT * FROM t LIMIT 5,0;",
+ count: 10,
+ want: "SELECT * FROM t LIMIT 5,0;",
+ },
{
stmt: "SELECT * FROM t LIMIT 0,\n1000000;",
count: 10,
@@ -264,7 +270,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {
}
for _, tc := range testCases {
- got := getStatementWithResultLimit(tc.stmt, tc.count)
+ got := statementWithResultLimit(tc.stmt, tc.count, "")
require.Equal(t, tc.want, got, tc.stmt)
}
}
diff --git a/backend/plugin/parser/tidb/limit.go b/backend/plugin/parser/tidb/limit.go
new file mode 100644
index 0000000000..f4fcbfc1c5
--- /dev/null
+++ b/backend/plugin/parser/tidb/limit.go
@@ -0,0 +1,87 @@
+package tidb
+
+import (
+ "fmt"
+ "log/slog"
+ "strings"
+
+ tidbast "github.com/pingcap/tidb/pkg/parser/ast"
+ "github.com/pingcap/tidb/pkg/parser/format"
+ tidbdriver "github.com/pingcap/tidb/pkg/types/parser_driver"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_TIDB, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for TiDB, which
+// takes no engineVersion.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := statementWithResultLimitInline(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
+
+func statementWithResultLimitInline(statement string, limit int) (string, error) {
+ stmtList, err := ParseTiDB(statement, "", "")
+ if err != nil {
+ return "", errors.Wrapf(err, "failed to parse tidb statement: %s", statement)
+ }
+ if len(stmtList) != 1 {
+ return "", errors.Errorf("expect one single statement in the query, %s", statement)
+ }
+ restoreFlags := format.DefaultRestoreFlags | format.RestoreStringWithoutDefaultCharset
+ stmt := stmtList[0]
+ switch stmt := stmt.(type) {
+ case *tidbast.SelectStmt:
+ if stmt.Limit != nil && stmt.Limit.Count != nil {
+ if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {
+ userLimit := int(v.GetInt64())
+ if limit < userLimit {
+ userLimit = limit
+ }
+ stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")
+ }
+ } else {
+ stmt.Limit = &tidbast.Limit{
+ Count: tidbast.NewValueExpr(int64(limit), "", ""),
+ }
+ }
+ var buffer strings.Builder
+ ctx := format.NewRestoreCtx(restoreFlags, &buffer)
+ if err := stmt.Restore(ctx); err != nil {
+ return "", err
+ }
+ return buffer.String(), nil
+ case *tidbast.SetOprStmt:
+ if stmt.Limit != nil && stmt.Limit.Count != nil {
+ if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {
+ userLimit := int(v.GetInt64())
+ if limit < userLimit {
+ userLimit = limit
+ }
+ stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")
+ }
+ } else {
+ stmt.Limit = &tidbast.Limit{
+ Count: tidbast.NewValueExpr(int64(limit), "", ""),
+ }
+ }
+ var buffer strings.Builder
+ ctx := format.NewRestoreCtx(restoreFlags, &buffer)
+ if err := stmt.Restore(ctx); err != nil {
+ return "", err
+ }
+ return buffer.String(), nil
+ default:
+ }
+ return statement, nil
+}
diff --git a/backend/plugin/db/tidb/query_test.go b/backend/plugin/parser/tidb/limit_test.go
similarity index 95%
rename from backend/plugin/db/tidb/query_test.go
rename to backend/plugin/parser/tidb/limit_test.go
index b80f5a232b..bebf7ecc55 100644
--- a/backend/plugin/db/tidb/query_test.go
+++ b/backend/plugin/parser/tidb/limit_test.go
@@ -55,7 +55,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {
}
for _, tc := range testCases {
- got := getStatementWithResultLimit(tc.stmt, tc.count)
+ got := statementWithResultLimit(tc.stmt, tc.count, "")
require.Equal(t, tc.want, got, tc.stmt)
}
}
diff --git a/backend/plugin/parser/tsql/limit.go b/backend/plugin/parser/tsql/limit.go
new file mode 100644
index 0000000000..a9d99d560b
--- /dev/null
+++ b/backend/plugin/parser/tsql/limit.go
@@ -0,0 +1,28 @@
+package tsql
+
+import (
+ "fmt"
+ "log/slog"
+
+ omnimssql "github.com/bytebase/omni/mssql"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func init() {
+ base.RegisterResultLimitFunc(storepb.Engine_MSSQL, statementWithResultLimit)
+}
+
+// statementWithResultLimit implements base.ResultLimitFunc for SQL Server,
+// which takes no engineVersion. The rewrite itself lives in omni, which already
+// owns the T-SQL grammar this needs.
+func statementWithResultLimit(statement string, limit int, _ string) string {
+ stmt, err := omnimssql.StatementWithResultLimit(statement, limit)
+ if err != nil {
+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))
+ return fmt.Sprintf("WITH result AS (%s) SELECT TOP %d * FROM result;", base.TrimStatement(statement), limit)
+ }
+ return stmt
+}
diff --git a/backend/plugin/db/mssql/query_test.go b/backend/plugin/parser/tsql/limit_test.go
similarity index 93%
rename from backend/plugin/db/mssql/query_test.go
rename to backend/plugin/parser/tsql/limit_test.go
index de93735150..aa0dc2944f 100644
--- a/backend/plugin/db/mssql/query_test.go
+++ b/backend/plugin/parser/tsql/limit_test.go
@@ -1,4 +1,4 @@
-package mssql
+package tsql
import (
"io"
@@ -35,7 +35,7 @@ func runLimitTest(t *testing.T, file string, record bool) {
require.NoError(t, err)
for i, tc := range testCases {
- want := getStatementWithResultLimit(tc.Stmt, tc.Limit)
+ want := statementWithResultLimit(tc.Stmt, tc.Limit, "")
if record {
testCases[i].Want = want
} else {
diff --git a/backend/plugin/db/mssql/test-data/test_limit.yaml b/backend/plugin/parser/tsql/test-data/test_limit.yaml
similarity index 100%
rename from backend/plugin/db/mssql/test-data/test_limit.yaml
rename to backend/plugin/parser/tsql/test-data/test_limit.yaml