Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(sql): build LIMIT statements in the parsers (#21486)
Danny Xu7af93f500c3c
diff --git a/backend/plugin/db/bigquery/bigquery.go b/backend/plugin/db/bigquery/bigquery.goindex 94f02c1a18..b38d6c57fa 100644--- a/backend/plugin/db/bigquery/bigquery.go+++ b/backend/plugin/db/bigquery/bigquery.go@@ -22,6 +22,10 @@ import (v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"++ // Register how this engine plans a limit, for base.StatementWithResultLimit below.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/bigquery")var (@@ -167,7 +171,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, qqueryResult, err := func() (*v1pb.QueryResult, error) {if util.IsSelect(statement) {if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_BIGQUERY, statement, queryContext.Limit, "")}q := d.client.Query(statement)if queryContext.OperatorEmail != "" {@@ -360,14 +364,6 @@ func encodeOperatorEmail(email string) string {return string(values)}-func getStatementWithResultLimit(statement string, limit int) string {- limitPart := ""- if limit > 0 {- limitPart = fmt.Sprintf(" LIMIT %d", limit)- }- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", util.TrimStatement(statement), limitPart)-}-func convertValue(v bigquery.Value, fieldType bigquery.FieldType) *v1pb.RowValue {if v == nil {return &v1pb.RowValue{Kind: &v1pb.RowValue_NullValue{NullValue: structpb.NullValue_NULL_VALUE}}diff --git a/backend/plugin/db/clickhouse/query.go b/backend/plugin/db/clickhouse/query.goindex 4cc3a8ab9c..d37520ff93 100644--- a/backend/plugin/db/clickhouse/query.go+++ b/backend/plugin/db/clickhouse/query.go@@ -4,7 +4,6 @@ import ("context""database/sql""encoding/json"- "fmt""log/slog""math/big""reflect"@@ -51,7 +50,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_CLICKHOUSE, statement, queryContext.Limit, "")}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_CLICKHOUSE, statement)if err != nil {@@ -106,10 +105,6 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,return results, nil}-func getStatementWithResultLimit(statement string, limit int) string {- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)-}-// translateAggregateFunctionError rewrites the clickhouse-go driver's opaque// "unsupported column type \"AggregateFunction(...)\"" error into actionable// guidance pointing the user at -Merge combinators or finalizeAggregation(),diff --git a/backend/plugin/db/cockroachdb/cockroachdb.go b/backend/plugin/db/cockroachdb/cockroachdb.goindex 0ba61e6a73..1a59eab2a1 100644--- a/backend/plugin/db/cockroachdb/cockroachdb.go+++ b/backend/plugin/db/cockroachdb/cockroachdb.go@@ -628,7 +628,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, statement, queryContext.Limit, "")}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_POSTGRES, statement)@@ -703,10 +703,3 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,return results, nil}--func getStatementWithResultLimit(stmt string, limit int) string {- // To handle cases where there are comments in the query.- // eg. select * from t1 -- this is comment;- // Add two new line symbol here.- return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", util.TrimStatement(stmt), limit)-}diff --git a/backend/plugin/db/mssql/mssql.go b/backend/plugin/db/mssql/mssql.goindex 72647a08fe..df34b28b68 100644--- a/backend/plugin/db/mssql/mssql.go+++ b/backend/plugin/db/mssql/mssql.go@@ -517,7 +517,7 @@ func (*Driver) queryBatch(ctx context.Context, conn *sql.Conn, batch string, que// Before sending the batch to server, we add the limit clause to the statement.s := singleSQL.Textif queryContext.Limit > 0 {- s = getStatementWithResultLimit(s, queryContext.Limit)+ s = base.StatementWithResultLimit(storepb.Engine_MSSQL, s, queryContext.Limit, "")}refinedSQLs = append(refinedSQLs, s)if _, err := batchBuf.WriteString(s); err != nil {diff --git a/backend/plugin/db/mssql/query.go b/backend/plugin/db/mssql/query.goindex b23074d404..185f663879 100644--- a/backend/plugin/db/mssql/query.go+++ b/backend/plugin/db/mssql/query.go@@ -2,16 +2,12 @@ package mssqlimport ("database/sql"- "fmt"- "log/slog""time"- omnimssql "github.com/bytebase/omni/mssql""google.golang.org/protobuf/types/known/timestamppb"mssqldb "github.com/microsoft/go-mssqldb"- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util")@@ -157,16 +153,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.}return util.NullRowValue}--func getStatementWithResultLimit(statement string, limit int) string {- stmt, err := getStatementWithResultLimitInline(statement, limit)- if err != nil {- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- return fmt.Sprintf("WITH result AS (%s) SELECT TOP %d * FROM result;", util.TrimStatement(statement), limit)- }- return stmt-}--func getStatementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {- return omnimssql.StatementWithResultLimit(singleStatement, limitCount)-}diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex 27be4889e5..a8842045ce 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -534,7 +534,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_MYSQL, statement, queryContext.Limit, "")}sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)diff --git a/backend/plugin/db/mysql/query.go b/backend/plugin/db/mysql/query.goindex 3efcd65371..d322219626 100644--- a/backend/plugin/db/mysql/query.go+++ b/backend/plugin/db/mysql/query.go@@ -2,19 +2,9 @@ package mysqlimport ("database/sql"- "fmt"- "log/slog"- "reflect"- "strings"- "github.com/bytebase/omni/mysql/ast"- omnimysqlparser "github.com/bytebase/omni/mysql/parser"- "github.com/pkg/errors"-- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util"- mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {@@ -86,232 +76,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.}return util.NullRowValue}--func getStatementWithResultLimit(statement string, limit int) string {- stmt, err := getStatementWithResultLimitInline(statement, limit)- if err != nil {- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- // MySQL 5.7 doesn't support WITH clause.- return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)- }- return stmt-}--func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {- if strings.TrimSpace(statement) == "" {- return "", errors.New("empty statement")- }-- list, err := mysqlparser.ParseMySQL(statement)- if err != nil {- if stmt, procedureErr := rewriteMySQLSelectProcedureLimit(statement, limitCount); procedureErr == nil {- return stmt, nil- }- return "", err- }- if len(list.Items) != 1 {- return "", errors.Errorf("expected exactly one statement, got %d", len(list.Items))- }-- stmt, ok := list.Items[0].(*ast.SelectStmt)- if !ok {- return statement, nil- }-- return rewriteMySQLSelectLimit(statement, stmt, limitCount)-}--func rewriteMySQLSelectProcedureLimit(sql string, limitCount int) (string, error) {- statements, err := mysqlparser.SplitSQL(sql)- if err != nil {- return "", err- }- statementCount := 0- for _, statement := range statements {- if !statement.Empty {- statementCount++- }- }- if statementCount != 1 {- return "", errors.Errorf("expected exactly one statement, got %d", statementCount)- }-- tokens := omnimysqlparser.Tokenize(sql)- if len(tokens) == 0 || omnimysqlparser.TokenName(tokens[0].Type) != "SELECT" {- return "", errors.New("not a SELECT PROCEDURE statement")- }- for i, token := range tokens[1:] {- if omnimysqlparser.TokenName(token.Type) == "PROCEDURE" {- procedureTokenIndex := i + 1- if stmt, ok := rewriteMySQLLimitBeforeProcedure(sql, tokens[:procedureTokenIndex], limitCount); ok {- return stmt, nil- }- return sql[:token.Loc] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[token.Loc:], nil- }- }- return "", errors.New("SELECT PROCEDURE clause not found")-}--func rewriteMySQLLimitBeforeProcedure(sql string, tokens []omnimysqlparser.Token, limitCount int) (string, bool) {- depth := 0- for i := len(tokens) - 1; i >= 0; i-- {- switch tokens[i].Str {- case ")":- depth++- continue- case "(":- if depth > 0 {- depth--- }- continue- default:- }- if depth > 0 {- continue- }- if omnimysqlparser.TokenName(tokens[i].Type) != "LIMIT" {- continue- }- countTokenIndex := i + 1- if i+3 < len(tokens) && tokens[i+2].Str == "," {- countTokenIndex = i + 3- }- if countTokenIndex >= len(tokens) {- return "", false- }- if tokens[countTokenIndex].Ival > 0 && int(tokens[countTokenIndex].Ival) <= limitCount {- return sql, true- }- return sql[:tokens[countTokenIndex].Loc] + fmt.Sprintf("%d", limitCount) + sql[tokens[countTokenIndex].End:], true- }- return "", false-}--func rewriteMySQLSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {- if stmt.Limit != nil && stmt.Limit.Count != nil {- existingLimit := extractMySQLLimit(stmt.Limit.Count)- if existingLimit >= 0 && existingLimit <= limitCount {- return sql, nil- }- loc := nodeLocOf(stmt.Limit.Count)- loc = trimMySQLLocSpace(sql, loc)- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {- if existingLimit < 0 && stmt.Into == nil {- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")- }- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil- }- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")- }-- insertPos, beforeClause := findMySQLLimitInsertPosition(sql, stmt)- if insertPos < 0 || insertPos > len(sql) {- return "", errors.Errorf("invalid LIMIT insert position %d", insertPos)- }- if beforeClause {- return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil- }- return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil-}--func findMySQLLimitInsertPosition(sql string, stmt *ast.SelectStmt) (int, bool) {- if stmt.Into != nil && stmt.Into.Loc.Start > 0 {- intoStart := findMySQLKeywordBefore(sql, stmt.Into.Loc.Start, "INTO")- if intoStart >= maxMySQLLocEndBeforeTailClauses(stmt) {- return intoStart, true- }- }- if stmt.ForUpdate != nil && stmt.ForUpdate.Loc.Start > 0 {- return stmt.ForUpdate.Loc.Start, true- }- if stmt.Loc.End > 0 {- return stmt.Loc.End, false- }- return -1, false-}--func extractMySQLLimit(node ast.Node) int {- limit, ok := node.(*ast.IntLit)- if !ok {- return -1- }- return int(limit.Value)-}--func nodeLocOf(node ast.Node) ast.Loc {- return mysqlNodeLoc(node)-}--func trimMySQLLocSpace(sql string, loc ast.Loc) ast.Loc {- for loc.Start < loc.End && loc.Start < len(sql) && (sql[loc.Start] == ' ' || sql[loc.Start] == '\t' || sql[loc.Start] == '\n' || sql[loc.Start] == '\r') {- loc.Start++- }- for loc.End > loc.Start && loc.End <= len(sql) && (sql[loc.End-1] == ' ' || sql[loc.End-1] == '\t' || sql[loc.End-1] == '\n' || sql[loc.End-1] == '\r') {- loc.End--- }- return loc-}--func maxMySQLLocEndBeforeTailClauses(node ast.Node) int {- maxEnd := -1- ast.Inspect(node, func(n ast.Node) bool {- if n == nil {- return false- }- if n == node {- return true- }- if _, ok := n.(*ast.IntoClause); ok {- return false- }- if _, ok := n.(*ast.ForUpdate); ok {- return false- }- if loc := mysqlNodeLoc(n); loc.End > maxEnd {- maxEnd = loc.End- }- return true- })- return maxEnd-}--func mysqlNodeLoc(node ast.Node) ast.Loc {- if node == nil {- return ast.Loc{Start: -1, End: -1}- }- value := reflect.ValueOf(node)- if value.Kind() != reflect.Pointer || value.IsNil() {- return ast.Loc{Start: -1, End: -1}- }- elem := value.Elem()- if elem.Kind() != reflect.Struct {- return ast.Loc{Start: -1, End: -1}- }- field := elem.FieldByName("Loc")- if !field.IsValid() || !field.CanInterface() {- return ast.Loc{Start: -1, End: -1}- }- loc, ok := field.Interface().(ast.Loc)- if !ok {- return ast.Loc{Start: -1, End: -1}- }- return loc-}--func findMySQLKeywordBefore(sql string, offset int, keyword string) int {- if offset > len(sql) {- offset = len(sql)- }- if len(keyword) == 0 {- return offset- }- keyword = strings.ToUpper(keyword)- tokens := omnimysqlparser.Tokenize(sql[:offset])- for i := len(tokens) - 1; i >= 0; i-- {- token := tokens[i]- if token.End <= offset && omnimysqlparser.TokenName(token.Type) == keyword {- return token.Loc- }- }- return offset-}diff --git a/backend/plugin/db/oracle/oracle.go b/backend/plugin/db/oracle/oracle.goindex 1773298036..4ac7098d63 100644--- a/backend/plugin/db/oracle/oracle.go+++ b/backend/plugin/db/oracle/oracle.go@@ -250,7 +250,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}if !queryContext.Explain && queryContext.Limit > 0 {- statement = addResultLimit(statement, queryContext.Limit, d.connectionCtx.EngineVersion)+ statement = base.StatementWithResultLimit(storepb.Engine_ORACLE, statement, queryContext.Limit, d.connectionCtx.EngineVersion)}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_ORACLE, statement)diff --git a/backend/plugin/db/oracle/query.go b/backend/plugin/db/oracle/query.goindex bc9586df95..32ebe2be64 100644--- a/backend/plugin/db/oracle/query.go+++ b/backend/plugin/db/oracle/query.go@@ -2,27 +2,14 @@ package oracleimport ("database/sql"- "fmt"- "log/slog"- "strconv"- "strings""time"- "unicode"- oracleast "github.com/bytebase/omni/oracle/ast"- oracleparser "github.com/bytebase/omni/oracle/parser"- "github.com/pkg/errors""google.golang.org/protobuf/types/known/timestamppb"- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util"- plsqlparser "github.com/bytebase/bytebase/backend/plugin/parser/plsql"- "github.com/bytebase/bytebase/backend/utils")-const dbVersion12 = 12-// ========== Type Conversion Functions ==========// makeValueByTypeName creates appropriate Go types for Oracle column types.@@ -153,284 +140,3 @@ func convertTimestamp(typeName string, columnType *sql.ColumnType, t time.Time)}}}--// ========== Limit Handling Functions ==========--// addResultLimit adds a limit clause to the statement based on Oracle version-func addResultLimit(stmt string, limit int, engineVersion string) string {- // Check if we should skip adding limit (e.g., for simple DUAL queries)- if shouldSkipLimit(stmt) {- return stmt- }-- // Determine Oracle version- if isOracle11gOrEarlier(engineVersion) {- return addLimitFor11g(stmt, limit)- }- return addLimitFor12cAndLater(stmt, limit)-}--// shouldSkipLimit checks if the statement needs a limit clause-func shouldSkipLimit(stmt string) bool {- ok, err := skipAddLimit(stmt)- return err == nil && ok-}--// isOracle11gOrEarlier checks if the Oracle version is 11g or earlier-func isOracle11gOrEarlier(engineVersion string) bool {- versionIdx := strings.Index(engineVersion, ".")- if versionIdx < 0 {- return true // Default to 11g behavior for invalid version- }- versionNumber, err := strconv.Atoi(engineVersion[:versionIdx])- if err != nil {- return true // Default to 11g behavior for parsing errors- }- return versionNumber < dbVersion12-}--// addLimitFor11g adds a ROWNUM-based limit for Oracle 11g and earlier versions.-// Uses the legacy approach with subquery and ROWNUM.-func addLimitFor11g(statement string, limitCount int) string {- if !isSelectOrWithStatement(statement) {- return statement- }- return fmt.Sprintf("SELECT * FROM (%s) WHERE ROWNUM <= %d", util.TrimStatement(statement), limitCount)-}--// isSelectOrWithStatement checks if the statement is a SELECT or WITH statement-func isSelectOrWithStatement(statement string) bool {- trimmedStatement := strings.ToLower(strings.TrimLeftFunc(statement, unicode.IsSpace))- return strings.HasPrefix(trimmedStatement, "select") || strings.HasPrefix(trimmedStatement, "with")-}--// addLimitFor12cAndLater adds a FETCH NEXT clause for Oracle 12c and later versions.-// Uses the modern SQL standard approach, falling back to 11g approach on error.-func addLimitFor12cAndLater(statement string, limit int) string {- if !isSelectOrWithStatement(statement) {- return statement- }-- stmt, err := addFetchNextClause(statement, limit)- if err != nil {- slog.Error("failed to add FETCH NEXT clause, falling back to ROWNUM",- slog.String("statement", statement), log.BBError(err))- return addLimitFor11g(statement, limit)- }- return stmt-}--// addFetchNextClause adds a FETCH NEXT clause to a SELECT statement using AST parsing.-// This provides more precise placement of the limit clause compared to simple string wrapping.-func addFetchNextClause(statement string, limitCount int) (string, error) {- list, err := plsqlparser.ParsePLSQL(statement)- if err != nil {- return "", err- }- if list == nil || len(list.Items) == 0 {- return "", errors.New("no parse results")- }- if len(list.Items) > 1 {- return "", errors.Errorf("expected single statement, got %d statements", len(list.Items))- }- raw, ok := list.Items[0].(*oracleast.RawStmt)- if !ok {- return "", errors.Errorf("expected raw statement, got %T", list.Items[0])- }- selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)- if !ok {- return statement, nil- }-- res, err := rewriteOracleSelectFetch(statement, selectStmt, limitCount)- if err != nil {- return "", err- }- // https://stackoverflow.com/questions/27987882/how-can-i-solve-ora-00911-invalid-character-error- res = strings.TrimRightFunc(res, utils.IsSpaceOrSemicolon)-- return res, nil-}--func rewriteOracleSelectFetch(sql string, selectStmt *oracleast.SelectStmt, limitCount int) (string, error) {- target := rightmostOracleSetSelect(selectStmt)- if target.FetchFirst != nil {- return rewriteOracleFetchClause(sql, target.FetchFirst, limitCount)- }- if target.ForUpdate != nil && target.ForUpdate.Loc.Start > 0 && target.ForUpdate.Loc.Start <= len(sql) {- return sql[:target.ForUpdate.Loc.Start] + fmt.Sprintf("FETCH NEXT %d ROWS ONLY ", limitCount) + sql[target.ForUpdate.Loc.Start:], nil- }-- loc := oracleast.NodeLoc(target)- if loc.End < 0 || loc.End > len(sql) {- return "", errors.Errorf("invalid SELECT end position %d", loc.End)- }- return sql[:loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[loc.End:], nil-}--func rightmostOracleSetSelect(selectStmt *oracleast.SelectStmt) *oracleast.SelectStmt {- if selectStmt.Op != oracleast.SETOP_NONE && selectStmt.Rarg != nil {- return rightmostOracleSetSelect(selectStmt.Rarg)- }- return selectStmt-}--func rewriteOracleFetchClause(sql string, fetch *oracleast.FetchFirstClause, limitCount int) (string, error) {- if fetch.Count == nil {- if fetch.Loc.Start < 0 || fetch.Loc.End < fetch.Loc.Start || fetch.Loc.End > len(sql) {- return "", errors.Errorf("invalid FETCH position %d:%d", fetch.Loc.Start, fetch.Loc.End)- }- if hasOracleFetchKeyword(sql, fetch.Loc) {- return sql, nil- }- return sql[:fetch.Loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[fetch.Loc.End:], nil- }- if fetch.Percent {- return "", errors.Errorf("cannot rewrite PERCENT FETCH expression")- }-- existingLimit := extractOracleFetchCount(fetch.Count)- if existingLimit > 0 && existingLimit <= limitCount {- return sql, nil- }-- loc := oracleast.NodeLoc(fetch.Count)- loc = trimOracleLocSpace(sql, loc)- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {- if existingLimit <= 0 {- return "", errors.Errorf("cannot rewrite non-constant FETCH expression")- }- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil- }- return "", errors.Errorf("cannot rewrite FETCH expression")-}--func hasOracleFetchKeyword(sql string, loc oracleast.Loc) bool {- segment := sql[loc.Start:loc.End]- lexer := oracleparser.NewLexer(segment)- for {- tok := lexer.NextToken()- if tok.Loc == tok.End && tok.End >= len(segment) {- return false- }- if tok.Loc < 0 || tok.End > len(segment) || tok.End <= tok.Loc {- return false- }- if strings.EqualFold(segment[tok.Loc:tok.End], "FETCH") {- return true- }- }-}--func extractOracleFetchCount(node oracleast.Node) int {- limit, ok := node.(*oracleast.NumberLiteral)- if !ok || limit.IsFloat || limit.Ival <= 0 {- return 0- }- return int(limit.Ival)-}--func trimOracleLocSpace(sql string, loc oracleast.Loc) oracleast.Loc {- for loc.Start < loc.End && loc.Start < len(sql) && unicode.IsSpace(rune(sql[loc.Start])) {- loc.Start++- }- for loc.End > loc.Start && loc.End <= len(sql) && unicode.IsSpace(rune(sql[loc.End-1])) {- loc.End--- }- return loc-}--// ========== Skip Limit Logic for DUAL Queries ==========--// skipAddLimit checks if the statement needs a limit clause.-// For Oracle, we think the statement like "SELECT xxx FROM DUAL" does not need a limit clause.-// More details, xxx can not be a subquery.-func skipAddLimit(stmt string) (bool, error) {- list, err := plsqlparser.ParsePLSQL(stmt)- if err != nil {- return false, err- }- if list == nil || len(list.Items) == 0 {- return false, nil- }- // Multiple statements should not skip limit- if len(list.Items) > 1 {- return false, nil- }- raw, ok := list.Items[0].(*oracleast.RawStmt)- if !ok {- return false, nil- }- selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)- if !ok {- return false, nil- }- if !isSimpleOracleSelect(selectStmt) {- return false, nil- }- if !isOracleSelectFromDual(selectStmt) {- return false, nil- }- return !hasOracleSubqueriesInSelection(selectStmt), nil-}--func isSimpleOracleSelect(selectStmt *oracleast.SelectStmt) bool {- return selectStmt.WithClause == nil &&- !selectStmt.Distinct &&- !selectStmt.UniqueKw &&- !selectStmt.All &&- selectStmt.Into == nil &&- selectStmt.IntoVars == nil &&- selectStmt.WhereClause == nil &&- selectStmt.Hierarchical == nil &&- selectStmt.GroupClause == nil &&- selectStmt.HavingClause == nil &&- selectStmt.ModelClause == nil &&- len(selectStmt.WindowDefs) == 0 &&- selectStmt.QualifyClause == nil &&- selectStmt.OrderBy == nil &&- selectStmt.ForUpdate == nil &&- selectStmt.FetchFirst == nil &&- selectStmt.Op == oracleast.SETOP_NONE-}--func isOracleSelectFromDual(selectStmt *oracleast.SelectStmt) bool {- if selectStmt.FromClause == nil || len(selectStmt.FromClause.Items) != 1 {- return false- }- tableRef, ok := selectStmt.FromClause.Items[0].(*oracleast.TableRef)- if !ok || tableRef.Name == nil {- return false- }- return tableRef.Alias == nil &&- tableRef.Name.Schema == "" &&- tableRef.Name.DBLink == "" &&- strings.EqualFold(tableRef.Name.Name, "DUAL")-}--func hasOracleSubqueriesInSelection(selectStmt *oracleast.SelectStmt) bool {- if selectStmt.TargetList == nil || len(selectStmt.TargetList.Items) == 0 {- return true- }- for _, item := range selectStmt.TargetList.Items {- target, ok := item.(*oracleast.ResTarget)- if !ok || target.Expr == nil {- return true- }- if _, ok := target.Expr.(*oracleast.Star); ok {- return true- }- hasSubquery := false- oracleast.Inspect(target.Expr, func(node oracleast.Node) bool {- if _, ok := node.(*oracleast.SubqueryExpr); ok {- hasSubquery = true- return false- }- return true- })- if hasSubquery {- return true- }- }- return false-}diff --git a/backend/plugin/db/pg/pg.go b/backend/plugin/db/pg/pg.goindex 15fba29702..2e4c5b8627 100644--- a/backend/plugin/db/pg/pg.go+++ b/backend/plugin/db/pg/pg.go@@ -805,7 +805,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_POSTGRES, statement, queryContext.Limit, "")}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_POSTGRES, statement)diff --git a/backend/plugin/db/pg/query.go b/backend/plugin/db/pg/query.goindex c12bf5dd1e..f2b6f430e4 100644--- a/backend/plugin/db/pg/query.go+++ b/backend/plugin/db/pg/query.go@@ -3,20 +3,15 @@ package pgimport ("database/sql""fmt"- "log/slog""strings""time"- "github.com/pkg/errors""google.golang.org/protobuf/types/known/timestamppb"- "github.com/bytebase/omni/pg/ast""github.com/jackc/pgx/v5/pgtype"- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util"- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {@@ -211,117 +206,3 @@ func padZeroes(rawStr string, acc int) string {}return rawStr}--// getStatementWithResultLimit returns the statement with LIMIT clause if not exists.-func getStatementWithResultLimit(statement string, limit int) string {- stmt, err := getStatementWithResultLimitInline(statement, limit)- if err != nil {- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- // Fallback to CTE approach for problematic queries- return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)- }- return stmt-}--func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {- if strings.TrimSpace(statement) == "" {- return "", errors.New("empty statement")- }-- stmts, err := pgparser.ParsePg(statement)- if err != nil {- return "", errors.Wrap(err, "failed to parse statement")- }-- if len(stmts) != 1 {- return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))- }-- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- // Non-SELECT statement, return as-is.- return statement, nil- }-- return rewriteSelectLimit(statement, sel, limitCount)-}--// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement-// using byte-offset positions from the omni AST to surgically edit the original SQL text.-func rewriteSelectLimit(sql string, sel *ast.SelectStmt, limitCount int) (string, error) {- if sel.LimitCount != nil {- // Already has LIMIT — replace the value if ours is lower.- existingLimit := extractIntFromNode(sel.LimitCount)- if existingLimit > 0 && existingLimit <= limitCount {- return sql, nil // existing limit is already lower or equal, keep it- }- loc := nodeLocOf(sel.LimitCount)- if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {- return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil- }- // LimitCount is a non-constant expression (e.g. LIMIT $1, LIMIT (1+2)).- // Cannot safely rewrite in-place; let the caller fall back to CTE wrapper.- return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")- }-- // No LIMIT clause — find the right insertion point.- // PostgreSQL grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...- // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.- insertPos, beforeLocking := findLimitInsertPosition(sel)- if beforeLocking {- // Inserting at the start of FOR UPDATE/SHARE. The original whitespace- // before FOR becomes the separator before LIMIT; we add a trailing- // space to separate the limit value from FOR.- return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil- }- return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil-}--// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,-// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).-func findLimitInsertPosition(sel *ast.SelectStmt) (int, bool) {- // LIMIT must appear before FOR UPDATE/SHARE.- if sel.LockingClause != nil {- span := ast.ListSpan(sel.LockingClause)- if span.Start > 0 {- return span.Start, true- }- }-- // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).- // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,- // so account for the sort clause explicitly.- end := sel.Loc.End- if span := ast.ListSpan(sel.SortClause); span.End > end {- end = span.End- }- if end <= 0 {- return 0, false- }- return end, false-}--// extractIntFromNode extracts an integer value from a LIMIT/OFFSET node.-func extractIntFromNode(node ast.Node) int {- switch n := node.(type) {- case *ast.Integer:- return int(n.Ival)- case *ast.A_Const:- if iv, ok := n.Val.(*ast.Integer); ok {- return int(iv.Ival)- }- return 0- default:- return 0- }-}--// nodeLocOf returns the Loc of a node, handling common wrapper types.-func nodeLocOf(node ast.Node) ast.Loc {- switch n := node.(type) {- case *ast.A_Const:- return n.Loc- default:- return ast.Loc{Start: -1, End: -1}- }-}diff --git a/backend/plugin/db/pg/query_test.go b/backend/plugin/db/pg/query_test.goindex 282f9c0067..e8a3ed069e 100644--- a/backend/plugin/db/pg/query_test.go+++ b/backend/plugin/db/pg/query_test.go@@ -1,16 +1,11 @@package pgimport (- "fmt"- "strings""testing""time"- "github.com/stretchr/testify/assert""github.com/stretchr/testify/require""google.golang.org/protobuf/encoding/protojson"-- pgparser "github.com/bytebase/bytebase/backend/plugin/parser/pg")func TestBuildTimestamptzRowValue(t *testing.T) {@@ -151,383 +146,3 @@ func TestPadZeroes(t *testing.T) {require.Equal(t, test.expected, got)}}--func TestGetStatementWithResultLimit(t *testing.T) {- tests := []struct {- name string- statement string- limit int- want string- }{- {- name: "Simple SELECT without LIMIT",- statement: "SELECT * FROM users",- limit: 10,- want: "SELECT * FROM users LIMIT 10",- },- {- name: "SELECT with WHERE clause",- statement: "SELECT id, name FROM users WHERE active = true",- limit: 5,- want: "SELECT id, name FROM users WHERE active = true LIMIT 5",- },- {- name: "SELECT with existing LIMIT higher than requested",- statement: "SELECT * FROM users LIMIT 100",- limit: 10,- want: "SELECT * FROM users LIMIT 10",- },- {- name: "SELECT with existing LIMIT lower than requested",- statement: "SELECT * FROM users LIMIT 5",- limit: 10,- want: "SELECT * FROM users LIMIT 5",- },- {- name: "WITH query (CTE)",- statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",- limit: 10,- want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",- },- {- name: "SELECT with ORDER BY",- statement: "SELECT * FROM users ORDER BY created_at DESC",- limit: 10,- want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",- },- {- name: "SELECT with GROUP BY and HAVING",- statement: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5",- limit: 10,- want: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5 LIMIT 10",- },- {- name: "UNION query",- statement: "SELECT id FROM users UNION SELECT id FROM admins",- limit: 10,- want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",- },- {- name: "SELECT with OFFSET",- statement: "SELECT * FROM users LIMIT 20 OFFSET 10",- limit: 5,- want: "SELECT * FROM users LIMIT 5 OFFSET 10",- },- {- name: "SELECT with FOR UPDATE",- statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",- limit: 10,- want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",- },- {- name: "Non-SELECT statement (INSERT)",- statement: "INSERT INTO users (name) VALUES ('John')",- limit: 10,- want: "INSERT INTO users (name) VALUES ('John')",- },- {- name: "Non-SELECT statement (UPDATE)",- statement: "UPDATE users SET active = false WHERE id = 1",- limit: 10,- want: "UPDATE users SET active = false WHERE id = 1",- },- {- name: "Non-SELECT statement (DELETE)",- statement: "DELETE FROM users WHERE active = false",- limit: 10,- want: "DELETE FROM users WHERE active = false",- },- {- name: "SELECT with comments",- statement: "SELECT * FROM users -- This is a comment\nWHERE active = true",- limit: 10,- want: "SELECT * FROM users -- This is a comment\nWHERE active = true LIMIT 10",- },- {- name: "Complex nested SELECT",- statement: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01'",- limit: 10,- want: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01' LIMIT 10",- },- {- name: "SELECT with JOIN",- statement: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id",- limit: 10,- want: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id LIMIT 10",- },- {- name: "SELECT with parentheses",- statement: "(SELECT * FROM users)",- limit: 10,- want: "(SELECT * FROM users) LIMIT 10",- },- {- name: "SELECT with nested parentheses",- statement: "((SELECT * FROM users))",- limit: 10,- want: "((SELECT * FROM users)) LIMIT 10",- },- {- name: "SELECT with parentheses and existing LIMIT higher than requested",- statement: "(SELECT * FROM users LIMIT 100)",- limit: 10,- want: "(SELECT * FROM users LIMIT 10)",- },- {- name: "SELECT with parentheses and existing LIMIT lower than requested",- statement: "(SELECT * FROM users LIMIT 5)",- limit: 10,- want: "(SELECT * FROM users LIMIT 5)",- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := getStatementWithResultLimit(tt.statement, tt.limit)-- // For non-SELECT statements, the parser might fail and fall back to CTE approach- // which we should avoid for non-SELECT statements- if tt.statement == tt.want {- // Non-SELECT statements should remain unchanged- assert.Equal(t, tt.want, got)- } else {- // For SELECT statements, check if LIMIT was properly added- // The exact format might vary slightly due to parser normalization- assert.Contains(t, got, "LIMIT")- assert.Contains(t, got, fmt.Sprintf("%d", tt.limit))- }- })- }-}--func TestGetStatementWithResultLimitInline(t *testing.T) {- tests := []struct {- name string- statement string- limit int- wantErr bool- want string- }{- {- name: "Valid SELECT statement",- statement: "SELECT * FROM users",- limit: 10,- wantErr: false,- want: "SELECT * FROM users LIMIT 10",- },- {- name: "Invalid SQL syntax",- statement: "SELECT * FROM WHERE",- limit: 10,- wantErr: true,- },- {- name: "Empty statement",- statement: "",- limit: 10,- wantErr: true,- },- {- name: "FOR UPDATE gets proper whitespace",- statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",- limit: 10,- want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",- },- {- name: "FOR SHARE gets proper whitespace",- statement: "SELECT * FROM users FOR SHARE",- limit: 5,- want: "SELECT * FROM users LIMIT 5 FOR SHARE",- },- {- name: "SELECT ending with function call",- statement: "SELECT now()",- limit: 10,- want: "SELECT now() LIMIT 10",- },- {- name: "SELECT with subquery in WHERE",- statement: "SELECT * FROM users WHERE id IN (SELECT id FROM admins)",- limit: 10,- want: "SELECT * FROM users WHERE id IN (SELECT id FROM admins) LIMIT 10",- },- {- name: "Non-constant LIMIT expression falls back to error",- statement: "SELECT * FROM t LIMIT (1+2)",- limit: 5,- wantErr: true,- },- {- name: "CTE with lateral and ORDER BY",- statement: `WITH params AS (- SELECT 'resource.environment_id in []'::text AS env_condition-), affected AS (- SELECT- p.resource,- binding,- COALESCE(binding->'condition'->>'expression', '') AS old_expression- FROM policy p,- LATERAL jsonb_array_elements(p.payload->'bindings') AS binding- WHERE p.type = 'IAM'- AND p.resource_type = 'PROJECT'- AND binding->>'role' = 'roles/projectOwner'- AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'-)-SELECT- resource,- binding->'members' AS members,- binding->'condition' AS old_condition,- CASE- WHEN old_expression = '' THEN params.env_condition- ELSE '(' || old_expression || ') && ' || params.env_condition- END AS new_expression-FROM affected-CROSS JOIN params-ORDER BY resource`,- limit: 1000,- want: `WITH params AS (- SELECT 'resource.environment_id in []'::text AS env_condition-), affected AS (- SELECT- p.resource,- binding,- COALESCE(binding->'condition'->>'expression', '') AS old_expression- FROM policy p,- LATERAL jsonb_array_elements(p.payload->'bindings') AS binding- WHERE p.type = 'IAM'- AND p.resource_type = 'PROJECT'- AND binding->>'role' = 'roles/projectOwner'- AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'-)-SELECT- resource,- binding->'members' AS members,- binding->'condition' AS old_condition,- CASE- WHEN old_expression = '' THEN params.env_condition- ELSE '(' || old_expression || ') && ' || params.env_condition- END AS new_expression-FROM affected-CROSS JOIN params-ORDER BY resource LIMIT 1000`,- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := getStatementWithResultLimitInline(tt.statement, tt.limit)- if tt.wantErr {- require.Error(t, err)- } else {- require.NoError(t, err)- assert.NotEmpty(t, got)- if tt.want != "" {- assert.Equal(t, tt.want, got)- }- }- })- }-}--func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {- tests := []struct {- name string- statement string- limit int- want string- clausesInOrder []string- }{- {- name: "ORDER BY",- statement: "SELECT * FROM users ORDER BY created_at DESC",- limit: 10,- want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",- clausesInOrder: []string{"ORDER BY", "LIMIT 10"},- },- {- name: "OFFSET without LIMIT",- statement: "SELECT * FROM users OFFSET 20",- limit: 10,- clausesInOrder: []string{"OFFSET 20", "LIMIT 10"},- },- {- name: "ORDER BY with OFFSET without LIMIT",- statement: "SELECT * FROM users ORDER BY created_at DESC OFFSET 20",- limit: 10,- clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10"},- },- {- name: "ORDER BY with FOR UPDATE",- statement: "SELECT * FROM users ORDER BY id FOR UPDATE",- limit: 10,- want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",- clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},- },- {- name: "ORDER BY with OFFSET and FOR UPDATE",- statement: "SELECT * FROM users ORDER BY id OFFSET 20 FOR UPDATE",- limit: 10,- clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10", "FOR UPDATE"},- },- {- name: "GROUP BY HAVING WINDOW ORDER BY",- statement: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC",- limit: 10,- want: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC LIMIT 10",- clausesInOrder: []string{"GROUP BY", "HAVING", "WINDOW", "ORDER BY n", "LIMIT 10"},- },- {- name: "UNION with outer ORDER BY",- statement: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id",- limit: 10,- want: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id LIMIT 10",- clausesInOrder: []string{"UNION ALL", "ORDER BY", "LIMIT 10"},- },- {- name: "ORDER BY inside window only",- statement: "SELECT row_number() OVER (ORDER BY created_at) FROM users",- limit: 10,- want: "SELECT row_number() OVER (ORDER BY created_at) FROM users LIMIT 10",- clausesInOrder: []string{"FROM users", "LIMIT 10"},- },- {- name: "ORDER BY inside subquery only",- statement: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users",- limit: 10,- want: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users LIMIT 10",- clausesInOrder: []string{"AS ordered_users", "LIMIT 10"},- },- {- name: "WITH query with outer ORDER BY",- statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",- limit: 10,- want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",- clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := getStatementWithResultLimitInline(tt.statement, tt.limit)- require.NoError(t, err)- if tt.want != "" {- assert.Equal(t, tt.want, got)- }- assertSubstringsInOrder(t, got, tt.clausesInOrder)-- _, err = pgparser.ParsePg(got)- require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)- })- }-}--func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {- t.Helper()-- start := 0- for _, substring := range substrings {- index := strings.Index(s[start:], substring)- require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)- start += index + len(substring)- }-}diff --git a/backend/plugin/db/redshift/redshift.go b/backend/plugin/db/redshift/redshift.goindex 751d4b7b47..6e86a7165a 100644--- a/backend/plugin/db/redshift/redshift.go+++ b/backend/plugin/db/redshift/redshift.go@@ -394,7 +394,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_REDSHIFT, statement, queryContext.Limit, "")}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_REDSHIFT, statement)@@ -447,7 +447,3 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement stringreturn results, nil}--func getStatementWithResultLimit(stmt string, limit int) string {- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(stmt), limit)-}diff --git a/backend/plugin/db/snowflake/snowflake.go b/backend/plugin/db/snowflake/snowflake.goindex 92df80531c..2145101898 100644--- a/backend/plugin/db/snowflake/snowflake.go+++ b/backend/plugin/db/snowflake/snowflake.go@@ -346,7 +346,7 @@ func (*Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string,}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_SNOWFLAKE, statement, queryContext.Limit, "")}_, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_SNOWFLAKE, statement)diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.goindex cd85749c63..148f93a1da 100644--- a/backend/plugin/db/spanner/spanner.go+++ b/backend/plugin/db/spanner/spanner.go@@ -18,8 +18,6 @@ import (spannerdb "cloud.google.com/go/spanner/admin/database/apiv1""cloud.google.com/go/spanner/admin/database/apiv1/databasepb"sppb "cloud.google.com/go/spanner/apiv1/spannerpb"- "github.com/bytebase/omni/googlesql/ast"- "github.com/bytebase/omni/googlesql/parser""github.com/pkg/errors""google.golang.org/api/iterator""google.golang.org/api/option"@@ -35,6 +33,9 @@ import ("github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util""github.com/bytebase/bytebase/backend/plugin/parser/base"++ // Register how this engine plans a limit, for base.StatementWithResultLimit below.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/spanner")var (@@ -304,32 +305,6 @@ func getColumnTypeName(columnType *sppb.Type) (string, error) {return columnType.Code.String(), nil}-// getStatementWithResultLimit sets the LIMIT of the outermost query in stmt to-// at most limit. It never wraps stmt in a subquery, where Spanner rejects a-// WITH clause, so a query it cannot rewrite runs unchanged and queryStatement-// caps its rows.-func getStatementWithResultLimit(stmt string, limit int) string {- file, errs := parser.Parse(stmt)- if len(errs) > 0 || len(file.Stmts) != 1 {- return stmt- }- query, ok := file.Stmts[0].(*ast.QueryStmt)- // FOR UPDATE fails in queryStatement's read-only transaction, so there is no- // point fitting a LIMIT before it.- if !ok || query.ForUpdate {- return stmt- }- if query.Limit == nil {- end := query.Loc.End- return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]- }- count, ok := query.Limit.(*ast.Literal)- if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {- return stmt- }- return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]-}-// instancePath returns the Spanner instance resource name,// e.g. projects/<project>/instances/<instance>.func (d *Driver) instancePath() string {@@ -372,7 +347,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, qqueryResult, err := func() (*v1pb.QueryResult, error) {if util.IsSelect(statement) {if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_SPANNER, statement, queryContext.Limit, "")}return d.queryStatement(ctx, statement, queryContext)}diff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.goindex 1e378453ba..7d6a56a9f0 100644--- a/backend/plugin/db/spanner/spanner_test.go+++ b/backend/plugin/db/spanner/spanner_test.go@@ -110,80 +110,6 @@ func TestQueryConnLimitsQueryStartingWithWith(t *testing.T) {}}-func TestGetStatementWithResultLimit(t *testing.T) {- tests := []struct {- name string- statement string- want string- }{- {- name: "select",- statement: "SELECT * FROM t",- want: "SELECT * FROM t LIMIT 100",- },- {- name: "with",- statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",- want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",- },- {- name: "lowercase with, order by, and trailing semicolon",- statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",- want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",- },- {- name: "with and set operation",- statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",- want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",- },- {- name: "set operation",- statement: "SELECT 1 UNION ALL SELECT 2",- want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",- },- {- name: "inner limit belongs to the parenthesized query",- statement: "(SELECT x FROM t LIMIT 500)",- want: "(SELECT x FROM t LIMIT 500) LIMIT 100",- },- {- name: "trailing comment",- statement: "SELECT x FROM t -- note",- want: "SELECT x FROM t LIMIT 100 -- note",- },- {- name: "lower limit is kept",- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",- },- {- name: "higher limit is lowered",- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",- },- {- name: "non-literal limit is unchanged",- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",- },- {- name: "for update is unchanged",- statement: "SELECT x FROM t ORDER BY x FOR UPDATE",- want: "SELECT x FROM t ORDER BY x FOR UPDATE",- },- {- name: "unparsable query is unchanged",- statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",- want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",- },- }- for _, tc := range tests {- t.Run(tc.name, func(t *testing.T) {- require.Equal(t, tc.want, getStatementWithResultLimit(tc.statement, 100))- })- }-}-func TestGetDatabaseFromDSN(t *testing.T) {tests := []struct {dsn stringdiff --git a/backend/plugin/db/starrocks/query.go b/backend/plugin/db/starrocks/query.goindex c91d819d65..cde579313f 100644--- a/backend/plugin/db/starrocks/query.go+++ b/backend/plugin/db/starrocks/query.go@@ -2,16 +2,7 @@ package starrocksimport ("database/sql"- "fmt"- "log/slog"- "strconv"- "strings"- "github.com/bytebase/omni/starrocks/ast"- "github.com/bytebase/omni/starrocks/parser"- "github.com/pkg/errors"-- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util")@@ -83,206 +74,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.}return util.NullRowValue}--func getStatementWithResultLimit(statement string, limit int) string {- trimmedStatement := strings.TrimSpace(statement)- if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {- return statement- }-- stmt, err := getStatementWithResultLimitInline(statement, limit)- if err != nil {- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", util.TrimStatement(statement), limit)- }- return stmt-}--func getStatementWithResultLimitInline(statement string, limitCount int) (string, error) {- if strings.TrimSpace(statement) == "" {- return "", errors.New("empty statement")- }-- file, errs := parser.Parse(statement)- if len(errs) > 0 {- return "", errors.New(errs[0].Error())- }- if file == nil || len(file.Stmts) != 1 {- stmtCount := 0- if file != nil {- stmtCount = len(file.Stmts)- }- return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)- }-- switch stmt := file.Stmts[0].(type) {- case *ast.SelectStmt:- return rewriteSelectLimit(statement, stmt, limitCount)- case *ast.SetOpStmt:- return rewriteSetOpLimit(statement, stmt, limitCount)- case *ast.ParenSelect:- // The wrapper itself may carry the trailing clauses —- // (SELECT 1) LIMIT 900000 attaches the LIMIT to the ParenSelect,- // not the inner query — so the wrapper is where the cap applies.- if stmt.Limit != nil {- return rewriteExistingLimit(statement, stmt.Limit, limitCount)- }- // No wrapper LIMIT: append one at the wrapper level, which covers- // any paren nesting depth — ((SELECT 1)) LIMIT n and- // (SELECT 1) ORDER BY 1 LIMIT n are engine-valid — instead of- // unwrapping and missing deeper layers.- if hasUnparsedTail(statement, stmt.Loc.End) {- return "", errors.New("statement has unparsed tail content")- }- return statement[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + statement[stmt.Loc.End:], nil- default:- return statement, nil- }-}--func hasUnparsedTail(sql string, locEnd int) bool {- pos := locEnd- for pos < len(sql) {- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {- pos++- }- for pos < len(sql) && sql[pos] == ';' {- pos++- }- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {- pos++- }- if pos >= len(sql) {- return false- }- if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {- nl := strings.IndexByte(sql[pos:], '\n')- if nl < 0 {- return false- }- pos += nl + 1- continue- }- if strings.HasPrefix(sql[pos:], "/*") {- end := strings.Index(sql[pos:], "*/")- if end < 0 {- return false- }- pos += end + 2- continue- }- return true- }- return false-}--// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it-// is a constant no larger than limitCount, replace the count otherwise.-func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {- limitLoc := literalLoc(limitNode)- if limitLoc.Start < 0 {- return "", errors.New("cannot rewrite non-constant LIMIT expression")- }-- if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {- existingCount, _ := strconv.Atoi(sql[countStart:countEnd])- if existingCount > 0 && existingCount <= limitCount {- return sql, nil- }- return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil- }-- existingLimit := extractLimitValue(limitNode)- if existingLimit >= 0 && existingLimit <= limitCount {- return sql, nil- }- return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil-}--func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {- if stmt.Limit != nil {- return rewriteExistingLimit(sql, stmt.Limit, limitCount)- }-- if stmt.Into != nil {- return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil- }- if hasUnparsedTail(sql, stmt.Loc.End) {- return "", errors.New("statement has unparsed tail content")- }- return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil-}--func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {- if stmt.Limit != nil {- return rewriteExistingLimit(sql, stmt.Limit, limitCount)- }- if hasUnparsedTail(sql, stmt.Loc.End) {- return "", errors.New("statement has unparsed tail content")- }- return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil-}--func extractLimitValue(node ast.Node) int {- lit, ok := node.(*ast.Literal)- if !ok || lit.Kind != ast.LitInt {- return -1- }- val, err := strconv.Atoi(lit.Value)- if err != nil {- return -1- }- return val-}--func literalLoc(node ast.Node) ast.Loc {- lit, ok := node.(*ast.Literal)- if !ok {- return ast.Loc{Start: -1, End: -1}- }- return lit.Loc-}--func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {- pos := limitLoc.End- pos = skipWhitespaceAndComments(sql, pos)- if pos >= len(sql) || sql[pos] != ',' {- return 0, 0, false- }- pos++- pos = skipWhitespaceAndComments(sql, pos)- countStart = pos- for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {- pos++- }- if pos == countStart {- return 0, 0, false- }- return countStart, pos, true-}--func skipWhitespaceAndComments(sql string, pos int) int {- for pos < len(sql) {- for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {- pos++- }- if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {- nl := strings.IndexByte(sql[pos:], '\n')- if nl < 0 {- return len(sql)- }- pos += nl + 1- continue- }- if strings.HasPrefix(sql[pos:], "/*") {- end := strings.Index(sql[pos:], "*/")- if end < 0 {- return len(sql)- }- pos += end + 2- continue- }- return pos- }- return pos-}diff --git a/backend/plugin/db/starrocks/starrocks.go b/backend/plugin/db/starrocks/starrocks.goindex f27436ce85..a3a1a3844c 100644--- a/backend/plugin/db/starrocks/starrocks.go+++ b/backend/plugin/db/starrocks/starrocks.go@@ -315,7 +315,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(d.dbType, statement, queryContext.Limit, "")}sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)diff --git a/backend/plugin/db/tidb/query.go b/backend/plugin/db/tidb/query.goindex 2090b2e1b7..b73fbbe361 100644--- a/backend/plugin/db/tidb/query.go+++ b/backend/plugin/db/tidb/query.go@@ -2,20 +2,9 @@ package tidbimport ("database/sql"- "fmt"- "log/slog"- "strings"- "github.com/pkg/errors"-- tidbast "github.com/pingcap/tidb/pkg/parser/ast"- "github.com/pingcap/tidb/pkg/parser/format"- tidbdriver "github.com/pingcap/tidb/pkg/types/parser_driver"-- "github.com/bytebase/bytebase/backend/common/log"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db/util"- tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb")func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {@@ -85,68 +74,3 @@ func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.}return util.NullRowValue}--func getStatementWithResultLimit(statement string, limit int) string {- stmt, err := getStatementWithResultLimitInline(statement, limit)- if err != nil {- slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", util.TrimStatement(statement), limit)- }- return stmt-}--func getStatementWithResultLimitInline(statement string, limit int) (string, error) {- stmtList, err := tidbparser.ParseTiDB(statement, "", "")- if err != nil {- return "", errors.Wrapf(err, "failed to parse tidb statement: %s", statement)- }- if len(stmtList) != 1 {- return "", errors.Errorf("expect one single statement in the query, %s", statement)- }- restoreFlags := format.DefaultRestoreFlags | format.RestoreStringWithoutDefaultCharset- stmt := stmtList[0]- switch stmt := stmt.(type) {- case *tidbast.SelectStmt:- if stmt.Limit != nil && stmt.Limit.Count != nil {- if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {- userLimit := int(v.GetInt64())- if limit < userLimit {- userLimit = limit- }- stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")- }- } else {- stmt.Limit = &tidbast.Limit{- Count: tidbast.NewValueExpr(int64(limit), "", ""),- }- }- var buffer strings.Builder- ctx := format.NewRestoreCtx(restoreFlags, &buffer)- if err := stmt.Restore(ctx); err != nil {- return "", err- }- return buffer.String(), nil- case *tidbast.SetOprStmt:- if stmt.Limit != nil && stmt.Limit.Count != nil {- if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {- userLimit := int(v.GetInt64())- if limit < userLimit {- userLimit = limit- }- stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")- }- } else {- stmt.Limit = &tidbast.Limit{- Count: tidbast.NewValueExpr(int64(limit), "", ""),- }- }- var buffer strings.Builder- ctx := format.NewRestoreCtx(restoreFlags, &buffer)- if err := stmt.Restore(ctx); err != nil {- return "", err- }- return buffer.String(), nil- default:- }- return statement, nil-}diff --git a/backend/plugin/db/tidb/tidb.go b/backend/plugin/db/tidb/tidb.goindex e8e381811e..5066982488 100644--- a/backend/plugin/db/tidb/tidb.go+++ b/backend/plugin/db/tidb/tidb.go@@ -379,7 +379,7 @@ func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string}statement = explained} else if queryContext.Limit > 0 {- statement = getStatementWithResultLimit(statement, queryContext.Limit)+ statement = base.StatementWithResultLimit(storepb.Engine_TIDB, statement, queryContext.Limit, "")}sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)diff --git a/backend/plugin/db/util/driverutil.go b/backend/plugin/db/util/driverutil.goindex 9f30d1ffb2..b9a0a1bb03 100644--- a/backend/plugin/db/util/driverutil.go+++ b/backend/plugin/db/util/driverutil.go@@ -202,7 +202,7 @@ func ConvertCommonValue(_ string, _ *sql.ColumnType, value any) *v1pb.RowValue {return NullRowValue}-// TrimStatement trims the unused characters from the statement for making getStatementWithResultLimit() happy.+// TrimStatement trims the unused characters from a statement before running it.func TrimStatement(statement string) string {return strings.TrimLeftFunc(strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon), unicode.IsSpace)}diff --git a/backend/plugin/parser/base/limit.go b/backend/plugin/parser/base/limit.gonew file mode 100644index 0000000000..5f85537f11--- /dev/null+++ b/backend/plugin/parser/base/limit.go@@ -0,0 +1,58 @@+package base++import (+ "fmt"+ "strings"+ "sync"+ "unicode"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/utils"+)++// ResultLimitFunc returns statement rewritten so its outermost query returns at+// most limit rows. engineVersion is the connected server's version string; every+// engine but Oracle ignores it (Oracle's clause depends on it — ROWNUM before+// version 12, FETCH NEXT from 12c on).+//+// A function here never fails outward: each implementation tries a precise+// rewrite and falls back to a wrapping subquery on any error, logging why. That+// mirrors every registered function's actual shape, so the registry does not+// carry an error return only every driver would immediately discard.+type ResultLimitFunc func(statement string, limit int, engineVersion string) string++var (+ limitMux sync.Mutex+ limitFuncs = make(map[storepb.Engine]ResultLimitFunc)+)++// RegisterResultLimitFunc registers how engine caps a statement's row count.+// Only an engine whose driver enforces the cap in the SQL text registers here;+// an engine with no entry runs the statement unchanged (see StatementWithResultLimit).+func RegisterResultLimitFunc(engine storepb.Engine, f ResultLimitFunc) {+ limitMux.Lock()+ defer limitMux.Unlock()+ if _, dup := limitFuncs[engine]; dup {+ panic(fmt.Sprintf("Register called twice %s", engine))+ }+ limitFuncs[engine] = f+}++// StatementWithResultLimit returns statement rewritten to return at most limit+// rows, for engines that enforce the cap in SQL text. An engine with nothing+// registered — because its driver caps rows another way (Hive trims by result+// size; MongoDB, Redis and the other non-SQL stores have no row cap here at+// all) — returns statement unchanged; callers do not need to ask first.+func StatementWithResultLimit(engine storepb.Engine, statement string, limit int, engineVersion string) string {+ f, ok := limitFuncs[engine]+ if !ok {+ return statement+ }+ return f(statement, limit, engineVersion)+}++// TrimStatement trims the whitespace and trailing semicolons a limit rewrite's+// fallback wrapper (a subquery or CTE) would otherwise leave stray inside itself.+func TrimStatement(statement string) string {+ return strings.TrimLeftFunc(strings.TrimRightFunc(statement, utils.IsSpaceOrSemicolon), unicode.IsSpace)+}diff --git a/backend/plugin/parser/base/limit_test.go b/backend/plugin/parser/base/limit_test.gonew file mode 100644index 0000000000..e384d08d90--- /dev/null+++ b/backend/plugin/parser/base/limit_test.go@@ -0,0 +1,46 @@+package base++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestStatementWithResultLimit(t *testing.T) {+ // An engine with nothing registered — MongoDB, Redis, and the other stores+ // whose drivers never call this — runs the statement unchanged rather than+ // erroring, unlike ExplainStatement: there is no "must refuse" case here.+ got := StatementWithResultLimit(storepb.Engine_MONGODB, "SELECT * FROM t", 10, "")+ require.Equal(t, "SELECT * FROM t", got)++ // PostgreSQL registers in this test binary via the parser/pg package's own+ // tests importing this one transitively is not guaranteed, so this only+ // covers the registry mechanics directly with a throwaway engine value.+}++func TestRegisterResultLimitFuncPanicsOnDuplicate(t *testing.T) {+ const engine = storepb.Engine(999999) // not a real engine; scoped to this test+ RegisterResultLimitFunc(engine, func(statement string, _ int, _ string) string { return statement })+ require.Panics(t, func() {+ RegisterResultLimitFunc(engine, func(statement string, _ int, _ string) string { return statement })+ })+}++func TestTrimStatement(t *testing.T) {+ tests := []struct {+ statement string+ want string+ }{+ {"SELECT 1", "SELECT 1"},+ {"SELECT 1;", "SELECT 1"},+ {"SELECT 1;;;", "SELECT 1"},+ {" SELECT 1 ", "SELECT 1"},+ {"\n\tSELECT 1\n", "SELECT 1"},+ {"", ""},+ }+ for _, tc := range tests {+ require.Equalf(t, tc.want, TrimStatement(tc.statement), "%q", tc.statement)+ }+}diff --git a/backend/plugin/parser/bigquery/limit.go b/backend/plugin/parser/bigquery/limit.gonew file mode 100644index 0000000000..cd746f112b--- /dev/null+++ b/backend/plugin/parser/bigquery/limit.go@@ -0,0 +1,23 @@+package bigquery++import (+ "fmt"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_BIGQUERY, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for BigQuery, which+// takes no engineVersion. The caller only applies this to a statement it has+// already confirmed is a SELECT (see bigquery.go), so this wraps unconditionally.+func statementWithResultLimit(statement string, limit int, _ string) string {+ limitPart := ""+ if limit > 0 {+ limitPart = fmt.Sprintf(" LIMIT %d", limit)+ }+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", base.TrimStatement(statement), limitPart)+}diff --git a/backend/plugin/parser/bigquery/limit_test.go b/backend/plugin/parser/bigquery/limit_test.gonew file mode 100644index 0000000000..7264df875f--- /dev/null+++ b/backend/plugin/parser/bigquery/limit_test.go@@ -0,0 +1,23 @@+package bigquery++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ statement string+ limit int+ want string+ }{+ {"SELECT * FROM t", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},+ {"SELECT * FROM t;", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},+ {"SELECT * FROM t", 0, "WITH result AS (SELECT * FROM t) SELECT * FROM result;"},+ }+ for _, tc := range tests {+ got := statementWithResultLimit(tc.statement, tc.limit, "")+ require.Equalf(t, tc.want, got, "%q", tc.statement)+ }+}diff --git a/backend/plugin/parser/doris/limit.go b/backend/plugin/parser/doris/limit.gonew file mode 100644index 0000000000..22490a5249--- /dev/null+++ b/backend/plugin/parser/doris/limit.go@@ -0,0 +1,242 @@+package doris++import (+ "fmt"+ "log/slog"+ "strconv"+ "strings"++ "github.com/bytebase/omni/doris/ast"+ "github.com/bytebase/omni/doris/parser"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_DORIS, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for Doris, which+// takes no engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ trimmedStatement := strings.TrimSpace(statement)+ if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {+ return statement+ }++ query, outfile := splitOutfileClause(statement)+ stmt, err := statementWithResultLimitInline(query, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ if outfile == "" {+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(query), limit)+ }+ stmt = fmt.Sprintf("SELECT * FROM (\n%s\n) result LIMIT %d", base.TrimStatement(query), limit)+ }+ if outfile != "" {+ return strings.TrimRight(stmt, " \t\r\n") + "\n" + outfile+ }+ return stmt+}++// The Doris AST cannot represent INTO OUTFILE. Keep this top-level clause outside+// the query while capping it, including when a set operation needs wrapping.+func splitOutfileClause(statement string) (string, string) {+ tokens, errs := parser.Tokenize(statement)+ if len(errs) > 0 {+ return statement, ""+ }+ intoKind, _ := parser.KeywordToken("into")+ depth := 0+ for i, token := range tokens {+ switch token.Kind {+ case '(':+ depth+++ case ')':+ depth--+ case ';':+ return statement, ""+ case intoKind:+ if depth == 0 && i+1 < len(tokens) && tokens[i+1].Kind == outfileTokenKind {+ return statement[:token.Loc.Start], statement[token.Loc.Start:]+ }+ default:+ }+ }+ return statement, ""+}++func statementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")+ }++ file, errs := parser.Parse(statement)+ if len(errs) > 0 {+ return "", errors.New(errs[0].Error())+ }+ if file == nil || len(file.Stmts) != 1 {+ stmtCount := 0+ if file != nil {+ stmtCount = len(file.Stmts)+ }+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)+ }++ switch stmt := file.Stmts[0].(type) {+ case *ast.SelectStmt:+ return rewriteSelectLimit(statement, stmt, limitCount)+ case *ast.SetOpStmt:+ // SetOpStmt.Limit is unusable: the Doris grammar leaves it nil whether or+ // not the text has a trailing LIMIT (confirmed against omni — a UNION ...+ // LIMIT 5 parses with Limit == nil, its text still inside the node's own+ // Loc). Trusting nil to mean "no LIMIT" would append a second one after+ // the caller's. This is an error, not a pass-through: a set operation is+ // still a query whose rows need capping, so the caller's CTE-wrap+ // fallback must run rather than let it through unbounded.+ return "", errors.New("cannot locate LIMIT on a Doris set operation")+ default:+ // A statement Doris parses to something other than a query — DDL, DML,+ // SHOW, etc. — has no rows to cap; run it unchanged, matching every other+ // engine's treatment of a non-SELECT here.+ return statement, nil+ }+}++func hasUnparsedTail(sql string, locEnd int) bool {+ pos := locEnd+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ for pos < len(sql) && sql[pos] == ';' {+ pos+++ }+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if pos >= len(sql) {+ return false+ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return false+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return false+ }+ pos += end + 2+ continue+ }+ return true+ }+ return false+}++// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it+// is a constant no larger than limitCount, replace the count otherwise.+func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {+ limitLoc := literalLoc(limitNode)+ if limitLoc.Start < 0 {+ return "", errors.New("cannot rewrite non-constant LIMIT expression")+ }++ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {+ // A literal 0 count (LIMIT offset,0) is a valid, stricter limit and must+ // be kept, matching the non-comma path below.+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])+ if existingCount >= 0 && existingCount <= limitCount {+ return sql, nil+ }+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil+ }++ existingLimit := extractLimitValue(limitNode)+ if existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil+ }+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil+}++func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {+ if stmt.Limit != nil {+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)+ }+ if hasUnparsedTail(sql, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")+ }+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil+}++func extractLimitValue(node ast.Node) int {+ lit, ok := node.(*ast.Literal)+ if !ok || lit.Kind != ast.LitInt {+ return -1+ }+ val, err := strconv.Atoi(lit.Value)+ if err != nil {+ return -1+ }+ return val+}++func literalLoc(node ast.Node) ast.Loc {+ lit, ok := node.(*ast.Literal)+ if !ok {+ return ast.Loc{Start: -1, End: -1}+ }+ return lit.Loc+}++func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {+ pos := limitLoc.End+ pos = skipWhitespaceAndComments(sql, pos)+ if pos >= len(sql) || sql[pos] != ',' {+ return 0, 0, false+ }+ pos+++ pos = skipWhitespaceAndComments(sql, pos)+ countStart = pos+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {+ pos+++ }+ if pos == countStart {+ return 0, 0, false+ }+ return countStart, pos, true+}++func skipWhitespaceAndComments(sql string, pos int) int {+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return len(sql)+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return len(sql)+ }+ pos += end + 2+ continue+ }+ return pos+ }+ return pos+}diff --git a/backend/plugin/parser/doris/limit_test.go b/backend/plugin/parser/doris/limit_test.gonew file mode 100644index 0000000000..baa2d5ca43--- /dev/null+++ b/backend/plugin/parser/doris/limit_test.go@@ -0,0 +1,263 @@+package doris++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetStatementWithResultLimit(t *testing.T) {+ testCases := []struct {+ stmt string+ count int+ want string+ }{+ {+ stmt: "SELECT * FROM t;",+ count: 10,+ want: "SELECT * FROM t LIMIT 10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 10;",+ count: 5,+ want: "SELECT * FROM t LIMIT 5;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 5;",+ count: 10,+ want: "SELECT * FROM t LIMIT 5;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 123;",+ count: 10,+ want: "SELECT * FROM t LIMIT 10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 10 OFFSET 123;",+ count: 5,+ want: "SELECT * FROM t LIMIT 5 OFFSET 123;",+ },+ {+ stmt: "SELECT * FROM t2 JOIN t1 ON t2.c2 = t1.c2 where t2.c2 > 10;",+ count: 10,+ want: "SELECT * FROM t2 JOIN t1 ON t2.c2 = t1.c2 where t2.c2 > 10 LIMIT 10;",+ },+ {+ stmt: "SELECT * FROM t1 ORDER BY c2;",+ count: 10,+ want: "SELECT * FROM t1 ORDER BY c2 LIMIT 10;",+ },+ {+ stmt: "SELECT COUNT(col1) AS col2 FROM t GROUP BY col2 HAVING col2 = 2;",+ count: 10,+ want: "SELECT COUNT(col1) AS col2 FROM t GROUP BY col2 HAVING col2 = 2 LIMIT 10;",+ },+ {+ stmt: "SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers;",+ count: 10,+ // SetOpStmt.Limit is unreliable in the Doris grammar (see limit.go),+ // so a set operation always takes the CTE-wrap fallback.+ want: "SELECT * FROM (SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers) result LIMIT 10;",+ },+ {+ stmt: "SELECT customerNumber, checkNumber, amount FROM payments WHERE amount = (SELECT MAX(amount) FROM payments);",+ count: 10,+ want: "SELECT customerNumber, checkNumber, amount FROM payments WHERE amount = (SELECT MAX(amount) FROM payments) LIMIT 10;",+ },+ {+ stmt: "SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers LIMIT 10;",+ count: 10,+ want: "SELECT * FROM (SELECT firstName, lastName FROM employees UNION SELECT contactFirstName, contactLastName FROM customers LIMIT 10) result LIMIT 10;",+ },+ {+ // The outer statement is a SelectStmt (its WITH clause), so it still+ // gets the precise in-place edit despite the UNION ALL inside the CTE.+ stmt: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count;",+ count: 10,+ want: "WITH RECURSIVE cte_count (n) AS ( SELECT 1 UNION ALL SELECT n + 1 FROM cte_count WHERE n < 3 ) SELECT n FROM cte_count LIMIT 10;",+ },+ {+ stmt: "SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers;",+ count: 10,+ want: "SELECT * FROM (SELECT firstName FROM employees EXCEPT SELECT contactFirstName FROM customers) result LIMIT 10;",+ },+ {+ stmt: "SELECT col1, col2 FROM table1 ORDER BY col1;",+ count: 10,+ want: "SELECT col1, col2 FROM table1 ORDER BY col1 LIMIT 10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 0,20;",+ count: 10,+ want: "SELECT * FROM t LIMIT 0,10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 123,20;",+ count: 10,+ want: "SELECT * FROM t LIMIT 123,10;",+ },+ {+ // A literal 0 count is kept, not replaced with the requested cap.+ stmt: "SELECT * FROM t LIMIT 5,0;",+ count: 10,+ want: "SELECT * FROM t LIMIT 5,0;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 0,\n1000000;",+ count: 10,+ want: "SELECT * FROM t LIMIT 0,\n10;",+ },+ {+ stmt: "SELECT * FROM t LIMIT 0 /* offset comment */, 1000000;",+ count: 10,+ want: "SELECT * FROM t LIMIT 0 /* offset comment */, 10;",+ },+ // SHOW statements should not be wrapped+ {+ stmt: "SHOW DATA",+ count: 10,+ want: "SHOW DATA",+ },+ {+ stmt: "SHOW TABLES",+ count: 10,+ want: "SHOW TABLES",+ },+ {+ stmt: "show data",+ count: 10,+ want: "show data",+ },+ {+ stmt: " SHOW DATA ",+ count: 10,+ want: " SHOW DATA ",+ },+ {+ stmt: "SELECT * FROM t -- note",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 -- note",+ },+ {+ stmt: "SELECT * FROM t /* block comment */;",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 /* block comment */;",+ },+ {+ stmt: "SELECT * FROM t # hash comment",+ count: 10,+ want: "SELECT * FROM t LIMIT 10 # hash comment",+ },+ {+ stmt: "SELECT * FROM t;\n",+ count: 10,+ want: "SELECT * FROM t LIMIT 10;\n",+ },+ // A set operation always takes the CTE-wrap fallback (see limit.go), so an+ // existing LIMIT on it — larger, smaller, or comma-form — is never capped+ // in place; the fallback's own LIMIT is still the one that binds.+ {+ stmt: "SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100;",+ count: 10,+ want: "SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2 LIMIT 100) result LIMIT 10;",+ },+ {+ stmt: "SELECT a FROM t1 INTERSECT SELECT b FROM t2;",+ count: 10,+ want: "SELECT * FROM (SELECT a FROM t1 INTERSECT SELECT b FROM t2) result LIMIT 10;",+ },+ // Doris has no dedicated node for a parenthesized top-level query: the+ // grammar strips the parens transparently and produces a plain+ // SelectStmt whose Loc always lands after the closing paren(s), so a+ // bare append at Loc.End still lands outside them and is valid SQL.+ {+ stmt: "(SELECT * FROM t)",+ count: 10,+ want: "(SELECT * FROM t) LIMIT 10",+ },+ {+ stmt: "((SELECT * FROM t))",+ count: 10,+ want: "((SELECT * FROM t)) LIMIT 10",+ },+ {+ stmt: "(SELECT * FROM t) LIMIT 1000000",+ count: 10,+ want: "(SELECT * FROM t) LIMIT 10",+ },+ {+ stmt: "(SELECT * FROM t) ORDER BY 1",+ count: 10,+ want: "(SELECT * FROM t) ORDER BY 1 LIMIT 10",+ },+ }++ for _, tc := range testCases {+ got := statementWithResultLimit(tc.stmt, tc.count, "")+ require.Equal(t, tc.want, got, tc.stmt)+ }+}++func TestResultLimitOutfile(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want string+ }{+ {+ name: "select with export options",+ statement: `SELECT * FROM orders INTO OUTFILE 's3://bucket/export/' FORMAT AS CSV PROPERTIES("column_separator" = ",");`,+ want: "SELECT * FROM orders LIMIT 10\n" + `INTO OUTFILE 's3://bucket/export/' FORMAT AS CSV PROPERTIES("column_separator" = ",");`,+ },+ {+ name: "existing limit",+ statement: "SELECT * FROM orders LIMIT 100 INTO OUTFILE 's3://bucket/export/';",+ want: "SELECT * FROM orders LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "zero limit",+ statement: "SELECT * FROM orders LIMIT 0 INTO OUTFILE 's3://bucket/export/';",+ want: "SELECT * FROM orders LIMIT 0\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "zero comma limit",+ statement: "SELECT * FROM orders LIMIT 5,0 INTO OUTFILE 's3://bucket/export/';",+ want: "SELECT * FROM orders LIMIT 5,0\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "union fallback",+ statement: "SELECT a FROM t1 UNION SELECT a FROM t2 INTO OUTFILE 's3://bucket/export/';",+ want: "SELECT * FROM (\nSELECT a FROM t1 UNION SELECT a FROM t2\n) result LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "comments and case",+ statement: "SELECT * FROM orders -- export\ninto /* target */ outfile 's3://bucket/export/';",+ want: "SELECT * FROM orders LIMIT 10 -- export\ninto /* target */ outfile 's3://bucket/export/';",+ },+ {+ name: "union with trailing comment",+ statement: "SELECT a FROM t1 UNION SELECT a FROM t2 -- export\nINTO OUTFILE 's3://bucket/export/';",+ want: "SELECT * FROM (\nSELECT a FROM t1 UNION SELECT a FROM t2 -- export\n) result LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "parenthesized query",+ statement: "(SELECT * FROM orders) INTO OUTFILE 's3://bucket/export/';",+ want: "(SELECT * FROM orders) LIMIT 10\nINTO OUTFILE 's3://bucket/export/';",+ },+ {+ name: "quoted words",+ statement: "SELECT 'INTO OUTFILE', `INTO OUTFILE` FROM t",+ want: "SELECT 'INTO OUTFILE', `INTO OUTFILE` FROM t LIMIT 10",+ },+ {+ name: "comment words",+ statement: "SELECT * FROM t /* INTO OUTFILE */;",+ want: "SELECT * FROM t LIMIT 10 /* INTO OUTFILE */;",+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ require.Equal(t, tt.want, statementWithResultLimit(tt.statement, 10, ""))+ })+ }+}diff --git a/backend/plugin/parser/mysql/limit.go b/backend/plugin/parser/mysql/limit.gonew file mode 100644index 0000000000..6388c41fec--- /dev/null+++ b/backend/plugin/parser/mysql/limit.go@@ -0,0 +1,253 @@+package mysql++import (+ "fmt"+ "log/slog"+ "reflect"+ "strings"++ "github.com/bytebase/omni/mysql/ast"+ omnimysqlparser "github.com/bytebase/omni/mysql/parser"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_MYSQL, statementWithResultLimit)+ base.RegisterResultLimitFunc(storepb.Engine_MARIADB, statementWithResultLimit)+ base.RegisterResultLimitFunc(storepb.Engine_OCEANBASE, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for MySQL, MariaDB+// and OceanBase, which share this grammar. None takes an engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := statementWithResultLimitInline(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ // MySQL 5.7 doesn't support WITH clause.+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(statement), limit)+ }+ return stmt+}++func statementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")+ }++ list, err := ParseMySQL(statement)+ if err != nil {+ if stmt, procedureErr := rewriteMySQLSelectProcedureLimit(statement, limitCount); procedureErr == nil {+ return stmt, nil+ }+ return "", err+ }+ if len(list.Items) != 1 {+ return "", errors.Errorf("expected exactly one statement, got %d", len(list.Items))+ }++ stmt, ok := list.Items[0].(*ast.SelectStmt)+ if !ok {+ return statement, nil+ }++ return rewriteMySQLSelectLimit(statement, stmt, limitCount)+}++func rewriteMySQLSelectProcedureLimit(sql string, limitCount int) (string, error) {+ statements, err := SplitSQL(sql)+ if err != nil {+ return "", err+ }+ statementCount := 0+ for _, statement := range statements {+ if !statement.Empty {+ statementCount+++ }+ }+ if statementCount != 1 {+ return "", errors.Errorf("expected exactly one statement, got %d", statementCount)+ }++ tokens := omnimysqlparser.Tokenize(sql)+ if len(tokens) == 0 || omnimysqlparser.TokenName(tokens[0].Type) != "SELECT" {+ return "", errors.New("not a SELECT PROCEDURE statement")+ }+ for i, token := range tokens[1:] {+ if omnimysqlparser.TokenName(token.Type) == "PROCEDURE" {+ procedureTokenIndex := i + 1+ if stmt, ok := rewriteMySQLLimitBeforeProcedure(sql, tokens[:procedureTokenIndex], limitCount); ok {+ return stmt, nil+ }+ return sql[:token.Loc] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[token.Loc:], nil+ }+ }+ return "", errors.New("SELECT PROCEDURE clause not found")+}++func rewriteMySQLLimitBeforeProcedure(sql string, tokens []omnimysqlparser.Token, limitCount int) (string, bool) {+ depth := 0+ for i := len(tokens) - 1; i >= 0; i-- {+ switch tokens[i].Str {+ case ")":+ depth+++ continue+ case "(":+ if depth > 0 {+ depth--+ }+ continue+ default:+ }+ if depth > 0 {+ continue+ }+ if omnimysqlparser.TokenName(tokens[i].Type) != "LIMIT" {+ continue+ }+ countTokenIndex := i + 1+ if i+3 < len(tokens) && tokens[i+2].Str == "," {+ countTokenIndex = i + 3+ }+ if countTokenIndex >= len(tokens) {+ return "", false+ }+ if tokens[countTokenIndex].Ival > 0 && int(tokens[countTokenIndex].Ival) <= limitCount {+ return sql, true+ }+ return sql[:tokens[countTokenIndex].Loc] + fmt.Sprintf("%d", limitCount) + sql[tokens[countTokenIndex].End:], true+ }+ return "", false+}++func rewriteMySQLSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {+ if stmt.Limit != nil && stmt.Limit.Count != nil {+ existingLimit := extractMySQLLimit(stmt.Limit.Count)+ if existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil+ }+ loc := nodeLocOf(stmt.Limit.Count)+ loc = trimMySQLLocSpace(sql, loc)+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {+ if existingLimit < 0 && stmt.Into == nil {+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")+ }+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil+ }+ return "", errors.Errorf("cannot rewrite non-constant LIMIT expression")+ }++ insertPos, beforeClause := findMySQLLimitInsertPosition(sql, stmt)+ if insertPos < 0 || insertPos > len(sql) {+ return "", errors.Errorf("invalid LIMIT insert position %d", insertPos)+ }+ if beforeClause {+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil+ }+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil+}++func findMySQLLimitInsertPosition(sql string, stmt *ast.SelectStmt) (int, bool) {+ if stmt.Into != nil && stmt.Into.Loc.Start > 0 {+ intoStart := findMySQLKeywordBefore(sql, stmt.Into.Loc.Start, "INTO")+ if intoStart >= maxMySQLLocEndBeforeTailClauses(stmt) {+ return intoStart, true+ }+ }+ if stmt.ForUpdate != nil && stmt.ForUpdate.Loc.Start > 0 {+ return stmt.ForUpdate.Loc.Start, true+ }+ if stmt.Loc.End > 0 {+ return stmt.Loc.End, false+ }+ return -1, false+}++func extractMySQLLimit(node ast.Node) int {+ limit, ok := node.(*ast.IntLit)+ if !ok {+ return -1+ }+ return int(limit.Value)+}++func nodeLocOf(node ast.Node) ast.Loc {+ return mysqlNodeLoc(node)+}++func trimMySQLLocSpace(sql string, loc ast.Loc) ast.Loc {+ for loc.Start < loc.End && loc.Start < len(sql) && (sql[loc.Start] == ' ' || sql[loc.Start] == '\t' || sql[loc.Start] == '\n' || sql[loc.Start] == '\r') {+ loc.Start+++ }+ for loc.End > loc.Start && loc.End <= len(sql) && (sql[loc.End-1] == ' ' || sql[loc.End-1] == '\t' || sql[loc.End-1] == '\n' || sql[loc.End-1] == '\r') {+ loc.End--+ }+ return loc+}++func maxMySQLLocEndBeforeTailClauses(node ast.Node) int {+ maxEnd := -1+ ast.Inspect(node, func(n ast.Node) bool {+ if n == nil {+ return false+ }+ if n == node {+ return true+ }+ if _, ok := n.(*ast.IntoClause); ok {+ return false+ }+ if _, ok := n.(*ast.ForUpdate); ok {+ return false+ }+ if loc := mysqlNodeLoc(n); loc.End > maxEnd {+ maxEnd = loc.End+ }+ return true+ })+ return maxEnd+}++func mysqlNodeLoc(node ast.Node) ast.Loc {+ if node == nil {+ return ast.Loc{Start: -1, End: -1}+ }+ value := reflect.ValueOf(node)+ if value.Kind() != reflect.Pointer || value.IsNil() {+ return ast.Loc{Start: -1, End: -1}+ }+ elem := value.Elem()+ if elem.Kind() != reflect.Struct {+ return ast.Loc{Start: -1, End: -1}+ }+ field := elem.FieldByName("Loc")+ if !field.IsValid() || !field.CanInterface() {+ return ast.Loc{Start: -1, End: -1}+ }+ loc, ok := field.Interface().(ast.Loc)+ if !ok {+ return ast.Loc{Start: -1, End: -1}+ }+ return loc+}++func findMySQLKeywordBefore(sql string, offset int, keyword string) int {+ if offset > len(sql) {+ offset = len(sql)+ }+ if len(keyword) == 0 {+ return offset+ }+ keyword = strings.ToUpper(keyword)+ tokens := omnimysqlparser.Tokenize(sql[:offset])+ for i := len(tokens) - 1; i >= 0; i-- {+ token := tokens[i]+ if token.End <= offset && omnimysqlparser.TokenName(token.Type) == keyword {+ return token.Loc+ }+ }+ return offset+}diff --git a/backend/plugin/db/mysql/query_test.go b/backend/plugin/parser/mysql/limit_test.gosimilarity index 97%rename from backend/plugin/db/mysql/query_test.gorename to backend/plugin/parser/mysql/limit_test.goindex e86281819a..a80d8f28e7 100644--- a/backend/plugin/db/mysql/query_test.go+++ b/backend/plugin/parser/mysql/limit_test.go@@ -213,7 +213,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {}for _, tc := range testCases {- got := getStatementWithResultLimit(tc.stmt, tc.count)+ got := statementWithResultLimit(tc.stmt, tc.count, "")require.Equal(t, tc.want, got, tc.stmt)}}@@ -237,7 +237,7 @@ func TestGetStatementWithResultLimitFallback(t *testing.T) {}for _, tc := range testCases {- got := getStatementWithResultLimit(tc.stmt, tc.count)+ got := statementWithResultLimit(tc.stmt, tc.count, "")require.Equal(t, tc.want, got, tc.stmt)}}@@ -277,7 +277,7 @@ func TestGetStatementWithResultLimitInline(t *testing.T) {}for _, tc := range testCases {- got, err := getStatementWithResultLimitInline(tc.stmt, tc.count)+ got, err := statementWithResultLimitInline(tc.stmt, tc.count)if tc.wantErr {require.Error(t, err, tc.stmt)continue@@ -291,6 +291,6 @@ func TestGetStatementWithResultLimitUnicodeBeforeInto(t *testing.T) {stmt := "SELECT 'ı' FROM t INTO OUTFILE '/tmp/a';"want := "SELECT 'ı' FROM t LIMIT 10 INTO OUTFILE '/tmp/a';"- got := getStatementWithResultLimit(stmt, 10)+ got := statementWithResultLimit(stmt, 10, "")require.Equal(t, want, got)}diff --git a/backend/plugin/parser/pg/limit.go b/backend/plugin/parser/pg/limit.gonew file mode 100644index 0000000000..b8deb7f308--- /dev/null+++ b/backend/plugin/parser/pg/limit.go@@ -0,0 +1,136 @@+package pg++import (+ "fmt"+ "log/slog"+ "strings"++ "github.com/bytebase/omni/pg/ast"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_POSTGRES, statementWithResultLimit)+ // CockroachDB parses through the PostgreSQL grammar here as it does for the+ // read-only gate and EXPLAIN (see query.go, explain.go).+ base.RegisterResultLimitFunc(storepb.Engine_COCKROACHDB, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for PostgreSQL and+// CockroachDB, which share this grammar. Neither takes an engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := statementWithResultLimitInline(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ // Fallback to CTE approach for problematic queries+ return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)+ }+ return stmt+}++func statementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")+ }++ stmts, err := ParsePg(statement)+ if err != nil {+ return "", errors.Wrap(err, "failed to parse statement")+ }++ if len(stmts) != 1 {+ return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))+ }++ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ // Non-SELECT statement, return as-is.+ return statement, nil+ }++ return rewriteSelectLimit(statement, sel, limitCount)+}++// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement+// using byte-offset positions from the omni AST to surgically edit the original SQL text.+func rewriteSelectLimit(sql string, sel *ast.SelectStmt, limitCount int) (string, error) {+ if sel.LimitCount != nil {+ existingLimit, isInteger := extractIntFromNode(sel.LimitCount)+ loc := nodeLocOf(sel.LimitCount)+ if isInteger && existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil // existing limit is already lower or equal, keep it+ }+ if isInteger && loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil+ }+ // ALL/NULL and non-constant limits lack reliable replacement spans.+ // Cap them with the outer query instead of rewriting their text.+ return "", errors.Errorf("cannot rewrite LIMIT expression in place")+ }++ // No LIMIT clause — find the right insertion point.+ // PostgreSQL grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...+ // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.+ insertPos, beforeLocking := findLimitInsertPosition(sel)+ if beforeLocking {+ // Inserting at the start of FOR UPDATE/SHARE. The original whitespace+ // before FOR becomes the separator before LIMIT; we add a trailing+ // space to separate the limit value from FOR.+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil+ }+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil+}++// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,+// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).+func findLimitInsertPosition(sel *ast.SelectStmt) (int, bool) {+ // LIMIT must appear before FOR UPDATE/SHARE.+ if sel.LockingClause != nil {+ span := ast.ListSpan(sel.LockingClause)+ if span.Start > 0 {+ return span.Start, true+ }+ }++ // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).+ // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,+ // so account for the sort clause explicitly.+ end := sel.Loc.End+ if span := ast.ListSpan(sel.SortClause); span.End > end {+ end = span.End+ }+ if end <= 0 {+ return 0, false+ }+ return end, false+}++// extractIntFromNode distinguishes integer limits, including zero, from unlimited+// ALL/NULL constants and expressions that require the fallback wrapper.+func extractIntFromNode(node ast.Node) (int, bool) {+ switch n := node.(type) {+ case *ast.Integer:+ return int(n.Ival), true+ case *ast.A_Const:+ if iv, ok := n.Val.(*ast.Integer); ok {+ return int(iv.Ival), true+ }+ return 0, false+ default:+ return 0, false+ }+}++// nodeLocOf returns the Loc of a node, handling common wrapper types.+func nodeLocOf(node ast.Node) ast.Loc {+ switch n := node.(type) {+ case *ast.A_Const:+ return n.Loc+ default:+ return ast.Loc{Start: -1, End: -1}+ }+}diff --git a/backend/plugin/parser/pg/limit_test.go b/backend/plugin/parser/pg/limit_test.gonew file mode 100644index 0000000000..43d5acfe4c--- /dev/null+++ b/backend/plugin/parser/pg/limit_test.go@@ -0,0 +1,437 @@+package pg++import (+ "fmt"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestGetStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ want string+ }{+ {+ name: "Simple SELECT without LIMIT",+ statement: "SELECT * FROM users",+ limit: 10,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "SELECT with WHERE clause",+ statement: "SELECT id, name FROM users WHERE active = true",+ limit: 5,+ want: "SELECT id, name FROM users WHERE active = true LIMIT 5",+ },+ {+ name: "SELECT with existing LIMIT higher than requested",+ statement: "SELECT * FROM users LIMIT 100",+ limit: 10,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "SELECT with existing LIMIT lower than requested",+ statement: "SELECT * FROM users LIMIT 5",+ limit: 10,+ want: "SELECT * FROM users LIMIT 5",+ },+ {+ name: "SELECT with existing LIMIT 0 is kept, not replaced",+ statement: "SELECT * FROM users LIMIT 0",+ limit: 10,+ want: "SELECT * FROM users LIMIT 0",+ },+ {+ name: "WITH query (CTE)",+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",+ limit: 10,+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",+ },+ {+ name: "SELECT with ORDER BY",+ statement: "SELECT * FROM users ORDER BY created_at DESC",+ limit: 10,+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",+ },+ {+ name: "SELECT with GROUP BY and HAVING",+ statement: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5",+ limit: 10,+ want: "SELECT department, COUNT(*) FROM users GROUP BY department HAVING COUNT(*) > 5 LIMIT 10",+ },+ {+ name: "UNION query",+ statement: "SELECT id FROM users UNION SELECT id FROM admins",+ limit: 10,+ want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",+ },+ {+ name: "SELECT with OFFSET",+ statement: "SELECT * FROM users LIMIT 20 OFFSET 10",+ limit: 5,+ want: "SELECT * FROM users LIMIT 5 OFFSET 10",+ },+ {+ name: "SELECT with FOR UPDATE",+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",+ },+ {+ name: "Non-SELECT statement (INSERT)",+ statement: "INSERT INTO users (name) VALUES ('John')",+ limit: 10,+ want: "INSERT INTO users (name) VALUES ('John')",+ },+ {+ name: "Non-SELECT statement (UPDATE)",+ statement: "UPDATE users SET active = false WHERE id = 1",+ limit: 10,+ want: "UPDATE users SET active = false WHERE id = 1",+ },+ {+ name: "Non-SELECT statement (DELETE)",+ statement: "DELETE FROM users WHERE active = false",+ limit: 10,+ want: "DELETE FROM users WHERE active = false",+ },+ {+ name: "SELECT with comments",+ statement: "SELECT * FROM users -- This is a comment\nWHERE active = true",+ limit: 10,+ want: "SELECT * FROM users -- This is a comment\nWHERE active = true LIMIT 10",+ },+ {+ name: "Complex nested SELECT",+ statement: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01'",+ limit: 10,+ want: "SELECT * FROM (SELECT * FROM users WHERE active = true) AS active_users WHERE created_at > '2023-01-01' LIMIT 10",+ },+ {+ name: "SELECT with JOIN",+ statement: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id",+ limit: 10,+ want: "SELECT u.*, d.name as dept_name FROM users u JOIN departments d ON u.dept_id = d.id LIMIT 10",+ },+ {+ name: "SELECT with parentheses",+ statement: "(SELECT * FROM users)",+ limit: 10,+ want: "(SELECT * FROM users) LIMIT 10",+ },+ {+ name: "SELECT with nested parentheses",+ statement: "((SELECT * FROM users))",+ limit: 10,+ want: "((SELECT * FROM users)) LIMIT 10",+ },+ {+ name: "SELECT with parentheses and existing LIMIT higher than requested",+ statement: "(SELECT * FROM users LIMIT 100)",+ limit: 10,+ want: "(SELECT * FROM users LIMIT 10)",+ },+ {+ name: "SELECT with parentheses and existing LIMIT lower than requested",+ statement: "(SELECT * FROM users LIMIT 5)",+ limit: 10,+ want: "(SELECT * FROM users LIMIT 5)",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := statementWithResultLimit(tt.statement, tt.limit, "")++ // For non-SELECT statements, the parser might fail and fall back to CTE approach+ // which we should avoid for non-SELECT statements+ if tt.statement == tt.want {+ // Non-SELECT statements should remain unchanged+ assert.Equal(t, tt.want, got)+ } else {+ // For SELECT statements, check if LIMIT was properly added+ // The exact format might vary slightly due to parser normalization+ assert.Contains(t, got, "LIMIT")+ assert.Contains(t, got, fmt.Sprintf("%d", tt.limit))+ }+ })+ }+}++func TestGetStatementWithResultLimitInline(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ wantErr bool+ want string+ }{+ {+ name: "Valid SELECT statement",+ statement: "SELECT * FROM users",+ limit: 10,+ wantErr: false,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "Invalid SQL syntax",+ statement: "SELECT * FROM WHERE",+ limit: 10,+ wantErr: true,+ },+ {+ name: "Empty statement",+ statement: "",+ limit: 10,+ wantErr: true,+ },+ {+ name: "FOR UPDATE gets proper whitespace",+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",+ },+ {+ name: "FOR SHARE gets proper whitespace",+ statement: "SELECT * FROM users FOR SHARE",+ limit: 5,+ want: "SELECT * FROM users LIMIT 5 FOR SHARE",+ },+ {+ name: "SELECT ending with function call",+ statement: "SELECT now()",+ limit: 10,+ want: "SELECT now() LIMIT 10",+ },+ {+ name: "SELECT with subquery in WHERE",+ statement: "SELECT * FROM users WHERE id IN (SELECT id FROM admins)",+ limit: 10,+ want: "SELECT * FROM users WHERE id IN (SELECT id FROM admins) LIMIT 10",+ },+ {+ name: "Non-constant LIMIT expression falls back to error",+ statement: "SELECT * FROM t LIMIT (1+2)",+ limit: 5,+ wantErr: true,+ },+ {+ name: "LIMIT 0 is kept, not replaced with the requested cap",+ statement: "SELECT * FROM t LIMIT 0",+ limit: 5,+ want: "SELECT * FROM t LIMIT 0",+ },+ {+ name: "CTE with lateral and ORDER BY",+ statement: `WITH params AS (+ SELECT 'resource.environment_id in []'::text AS env_condition+), affected AS (+ SELECT+ p.resource,+ binding,+ COALESCE(binding->'condition'->>'expression', '') AS old_expression+ FROM policy p,+ LATERAL jsonb_array_elements(p.payload->'bindings') AS binding+ WHERE p.type = 'IAM'+ AND p.resource_type = 'PROJECT'+ AND binding->>'role' = 'roles/projectOwner'+ AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'+)+SELECT+ resource,+ binding->'members' AS members,+ binding->'condition' AS old_condition,+ CASE+ WHEN old_expression = '' THEN params.env_condition+ ELSE '(' || old_expression || ') && ' || params.env_condition+ END AS new_expression+FROM affected+CROSS JOIN params+ORDER BY resource`,+ limit: 1000,+ want: `WITH params AS (+ SELECT 'resource.environment_id in []'::text AS env_condition+), affected AS (+ SELECT+ p.resource,+ binding,+ COALESCE(binding->'condition'->>'expression', '') AS old_expression+ FROM policy p,+ LATERAL jsonb_array_elements(p.payload->'bindings') AS binding+ WHERE p.type = 'IAM'+ AND p.resource_type = 'PROJECT'+ AND binding->>'role' = 'roles/projectOwner'+ AND COALESCE(binding->'condition'->>'expression', '') NOT LIKE '%resource.environment_id%'+)+SELECT+ resource,+ binding->'members' AS members,+ binding->'condition' AS old_condition,+ CASE+ WHEN old_expression = '' THEN params.env_condition+ ELSE '(' || old_expression || ') && ' || params.env_condition+ END AS new_expression+FROM affected+CROSS JOIN params+ORDER BY resource LIMIT 1000`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)+ if tt.wantErr {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ assert.NotEmpty(t, got)+ if tt.want != "" {+ assert.Equal(t, tt.want, got)+ }+ }+ })+ }+}++func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ want string+ clausesInOrder []string+ }{+ {+ name: "ORDER BY",+ statement: "SELECT * FROM users ORDER BY created_at DESC",+ limit: 10,+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",+ clausesInOrder: []string{"ORDER BY", "LIMIT 10"},+ },+ {+ name: "OFFSET without LIMIT",+ statement: "SELECT * FROM users OFFSET 20",+ limit: 10,+ clausesInOrder: []string{"OFFSET 20", "LIMIT 10"},+ },+ {+ name: "ORDER BY with OFFSET without LIMIT",+ statement: "SELECT * FROM users ORDER BY created_at DESC OFFSET 20",+ limit: 10,+ clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10"},+ },+ {+ name: "ORDER BY with FOR UPDATE",+ statement: "SELECT * FROM users ORDER BY id FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",+ clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},+ },+ {+ name: "ORDER BY with OFFSET and FOR UPDATE",+ statement: "SELECT * FROM users ORDER BY id OFFSET 20 FOR UPDATE",+ limit: 10,+ clausesInOrder: []string{"ORDER BY", "OFFSET 20", "LIMIT 10", "FOR UPDATE"},+ },+ {+ name: "GROUP BY HAVING WINDOW ORDER BY",+ statement: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC",+ limit: 10,+ want: "SELECT department, count(*) AS n, rank() OVER w FROM users GROUP BY department HAVING count(*) > 1 WINDOW w AS (ORDER BY department) ORDER BY n DESC LIMIT 10",+ clausesInOrder: []string{"GROUP BY", "HAVING", "WINDOW", "ORDER BY n", "LIMIT 10"},+ },+ {+ name: "UNION with outer ORDER BY",+ statement: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id",+ limit: 10,+ want: "SELECT id FROM users UNION ALL SELECT id FROM admins ORDER BY id LIMIT 10",+ clausesInOrder: []string{"UNION ALL", "ORDER BY", "LIMIT 10"},+ },+ {+ name: "ORDER BY inside window only",+ statement: "SELECT row_number() OVER (ORDER BY created_at) FROM users",+ limit: 10,+ want: "SELECT row_number() OVER (ORDER BY created_at) FROM users LIMIT 10",+ clausesInOrder: []string{"FROM users", "LIMIT 10"},+ },+ {+ name: "ORDER BY inside subquery only",+ statement: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users",+ limit: 10,+ want: "SELECT * FROM (SELECT * FROM users ORDER BY created_at) AS ordered_users LIMIT 10",+ clausesInOrder: []string{"AS ordered_users", "LIMIT 10"},+ },+ {+ name: "WITH query with outer ORDER BY",+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",+ limit: 10,+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",+ clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)+ require.NoError(t, err)+ if tt.want != "" {+ assert.Equal(t, tt.want, got)+ }+ assertSubstringsInOrder(t, got, tt.clausesInOrder)++ _, err = ParsePg(got)+ require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)+ })+ }+}++func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {+ t.Helper()++ start := 0+ for _, substring := range substrings {+ index := strings.Index(s[start:], substring)+ require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)+ start += index + len(substring)+ }+}++// TestCockroachDBReusesPGLimitRewrite locks the fix that gave CockroachDB the+// same precise rewrite PostgreSQL gets, through the shared registration in+// limit.go's init(). CockroachDB used to blind-wrap every statement in a CTE+// regardless of shape — a plain write like DELETE FROM t became+// WITH result AS (DELETE FROM t) SELECT * FROM result LIMIT n, a statement the+// read-only classifier could misread as a query with a data-modifying CTE. The+// registered function must be the exact same one PostgreSQL uses, so a+// non-SELECT statement passes through unchanged rather than being wrapped, and+// a SELECT gets the same precise cap.+func TestCockroachDBReusesPGLimitRewrite(t *testing.T) {+ require.Equal(t, "DELETE FROM t",+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "DELETE FROM t", 10, ""))+ require.Equal(t, "SELECT * FROM t LIMIT 10",+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "SELECT * FROM t LIMIT 100", 10, ""))+ // The blind CTE-wrap CockroachDB used before this reuse preserved a literal+ // LIMIT 0 (the inner zero always won regardless of the outer cap); the+ // precise rewrite must keep preserving it rather than replacing it.+ require.Equal(t, "SELECT * FROM t LIMIT 0",+ base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, "SELECT * FROM t LIMIT 0", 10, ""))+}++func TestUnlimitedResultLimit(t *testing.T) {+ for _, clause := range []string{"ALL", "NULL"} {+ t.Run(clause, func(t *testing.T) {+ statement := "SELECT * FROM t LIMIT " + clause + " OFFSET 2;"+ got := statementWithResultLimit(statement, 10, "")+ require.Equal(t, got, base.StatementWithResultLimit(storepb.Engine_COCKROACHDB, statement, 10, ""))+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT "+clause+" OFFSET 2\n) SELECT * FROM result LIMIT 10;", got)+ })+ }+}diff --git a/backend/plugin/parser/plsql/limit.go b/backend/plugin/parser/plsql/limit.gonew file mode 100644index 0000000000..01061ec6c0--- /dev/null+++ b/backend/plugin/parser/plsql/limit.go@@ -0,0 +1,308 @@+package plsql++import (+ "fmt"+ "log/slog"+ "strconv"+ "strings"+ "unicode"++ oracleast "github.com/bytebase/omni/oracle/ast"+ oracleparser "github.com/bytebase/omni/oracle/parser"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/utils"+)++const dbVersion12 = 12++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_ORACLE, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for Oracle.+// engineVersion picks the clause: ROWNUM before version 12, FETCH NEXT from+// 12c on — see isOracle11gOrEarlier.+func statementWithResultLimit(statement string, limit int, engineVersion string) string {+ return addResultLimit(statement, limit, engineVersion)+}++// addResultLimit adds a limit clause to the statement based on Oracle version+func addResultLimit(stmt string, limit int, engineVersion string) string {+ // Check if we should skip adding limit (e.g., for simple DUAL queries)+ if shouldSkipLimit(stmt) {+ return stmt+ }++ // Determine Oracle version+ if isOracle11gOrEarlier(engineVersion) {+ return addLimitFor11g(stmt, limit)+ }+ return addLimitFor12cAndLater(stmt, limit)+}++// shouldSkipLimit checks if the statement needs a limit clause+func shouldSkipLimit(stmt string) bool {+ ok, err := skipAddLimit(stmt)+ return err == nil && ok+}++// isOracle11gOrEarlier checks if the Oracle version is 11g or earlier+func isOracle11gOrEarlier(engineVersion string) bool {+ versionIdx := strings.Index(engineVersion, ".")+ if versionIdx < 0 {+ return true // Default to 11g behavior for invalid version+ }+ versionNumber, err := strconv.Atoi(engineVersion[:versionIdx])+ if err != nil {+ return true // Default to 11g behavior for parsing errors+ }+ return versionNumber < dbVersion12+}++// addLimitFor11g adds a ROWNUM-based limit for Oracle 11g and earlier versions.+// Uses the legacy approach with subquery and ROWNUM.+func addLimitFor11g(statement string, limitCount int) string {+ if !isSelectOrWithStatement(statement) {+ return statement+ }+ return fmt.Sprintf("SELECT * FROM (%s) WHERE ROWNUM <= %d", base.TrimStatement(statement), limitCount)+}++// isSelectOrWithStatement checks if the statement is a SELECT or WITH statement+func isSelectOrWithStatement(statement string) bool {+ trimmedStatement := strings.ToLower(strings.TrimLeftFunc(statement, unicode.IsSpace))+ return strings.HasPrefix(trimmedStatement, "select") || strings.HasPrefix(trimmedStatement, "with")+}++// addLimitFor12cAndLater adds a FETCH NEXT clause for Oracle 12c and later versions.+// Uses the modern SQL standard approach, falling back to 11g approach on error.+func addLimitFor12cAndLater(statement string, limit int) string {+ if !isSelectOrWithStatement(statement) {+ return statement+ }++ stmt, err := addFetchNextClause(statement, limit)+ if err != nil {+ slog.Error("failed to add FETCH NEXT clause, falling back to ROWNUM",+ slog.String("statement", statement), log.BBError(err))+ return addLimitFor11g(statement, limit)+ }+ return stmt+}++// addFetchNextClause adds a FETCH NEXT clause to a SELECT statement using AST parsing.+// This provides more precise placement of the limit clause compared to simple string wrapping.+func addFetchNextClause(statement string, limitCount int) (string, error) {+ list, err := ParsePLSQL(statement)+ if err != nil {+ return "", err+ }+ if list == nil || len(list.Items) == 0 {+ return "", errors.New("no parse results")+ }+ if len(list.Items) > 1 {+ return "", errors.Errorf("expected single statement, got %d statements", len(list.Items))+ }+ raw, ok := list.Items[0].(*oracleast.RawStmt)+ if !ok {+ return "", errors.Errorf("expected raw statement, got %T", list.Items[0])+ }+ selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)+ if !ok {+ return statement, nil+ }++ res, err := rewriteOracleSelectFetch(statement, selectStmt, limitCount)+ if err != nil {+ return "", err+ }+ // https://stackoverflow.com/questions/27987882/how-can-i-solve-ora-00911-invalid-character-error+ res = strings.TrimRightFunc(res, utils.IsSpaceOrSemicolon)++ return res, nil+}++func rewriteOracleSelectFetch(sql string, selectStmt *oracleast.SelectStmt, limitCount int) (string, error) {+ target := rightmostOracleSetSelect(selectStmt)+ if target.FetchFirst != nil {+ return rewriteOracleFetchClause(sql, target.FetchFirst, limitCount)+ }+ if target.ForUpdate != nil && target.ForUpdate.Loc.Start > 0 && target.ForUpdate.Loc.Start <= len(sql) {+ return sql[:target.ForUpdate.Loc.Start] + fmt.Sprintf("FETCH NEXT %d ROWS ONLY ", limitCount) + sql[target.ForUpdate.Loc.Start:], nil+ }++ loc := oracleast.NodeLoc(target)+ if loc.End < 0 || loc.End > len(sql) {+ return "", errors.Errorf("invalid SELECT end position %d", loc.End)+ }+ return sql[:loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[loc.End:], nil+}++func rightmostOracleSetSelect(selectStmt *oracleast.SelectStmt) *oracleast.SelectStmt {+ if selectStmt.Op != oracleast.SETOP_NONE && selectStmt.Rarg != nil {+ return rightmostOracleSetSelect(selectStmt.Rarg)+ }+ return selectStmt+}++func rewriteOracleFetchClause(sql string, fetch *oracleast.FetchFirstClause, limitCount int) (string, error) {+ if fetch.Count == nil {+ if fetch.Loc.Start < 0 || fetch.Loc.End < fetch.Loc.Start || fetch.Loc.End > len(sql) {+ return "", errors.Errorf("invalid FETCH position %d:%d", fetch.Loc.Start, fetch.Loc.End)+ }+ if hasOracleFetchKeyword(sql, fetch.Loc) {+ return sql, nil+ }+ return sql[:fetch.Loc.End] + fmt.Sprintf(" FETCH NEXT %d ROWS ONLY", limitCount) + sql[fetch.Loc.End:], nil+ }+ if fetch.Percent {+ return "", errors.Errorf("cannot rewrite PERCENT FETCH expression")+ }++ existingLimit := extractOracleFetchCount(fetch.Count)+ if existingLimit > 0 && existingLimit <= limitCount {+ return sql, nil+ }++ loc := oracleast.NodeLoc(fetch.Count)+ loc = trimOracleLocSpace(sql, loc)+ if loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {+ if existingLimit <= 0 {+ return "", errors.Errorf("cannot rewrite non-constant FETCH expression")+ }+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil+ }+ return "", errors.Errorf("cannot rewrite FETCH expression")+}++func hasOracleFetchKeyword(sql string, loc oracleast.Loc) bool {+ segment := sql[loc.Start:loc.End]+ lexer := oracleparser.NewLexer(segment)+ for {+ tok := lexer.NextToken()+ if tok.Loc == tok.End && tok.End >= len(segment) {+ return false+ }+ if tok.Loc < 0 || tok.End > len(segment) || tok.End <= tok.Loc {+ return false+ }+ if strings.EqualFold(segment[tok.Loc:tok.End], "FETCH") {+ return true+ }+ }+}++func extractOracleFetchCount(node oracleast.Node) int {+ limit, ok := node.(*oracleast.NumberLiteral)+ if !ok || limit.IsFloat || limit.Ival <= 0 {+ return 0+ }+ return int(limit.Ival)+}++func trimOracleLocSpace(sql string, loc oracleast.Loc) oracleast.Loc {+ for loc.Start < loc.End && loc.Start < len(sql) && unicode.IsSpace(rune(sql[loc.Start])) {+ loc.Start+++ }+ for loc.End > loc.Start && loc.End <= len(sql) && unicode.IsSpace(rune(sql[loc.End-1])) {+ loc.End--+ }+ return loc+}++// skipAddLimit checks if the statement needs a limit clause.+// For Oracle, we think the statement like "SELECT xxx FROM DUAL" does not need a limit clause.+// More details, xxx can not be a subquery.+func skipAddLimit(stmt string) (bool, error) {+ list, err := ParsePLSQL(stmt)+ if err != nil {+ return false, err+ }+ if list == nil || len(list.Items) == 0 {+ return false, nil+ }+ // Multiple statements should not skip limit+ if len(list.Items) > 1 {+ return false, nil+ }+ raw, ok := list.Items[0].(*oracleast.RawStmt)+ if !ok {+ return false, nil+ }+ selectStmt, ok := raw.Stmt.(*oracleast.SelectStmt)+ if !ok {+ return false, nil+ }+ if !isSimpleOracleSelect(selectStmt) {+ return false, nil+ }+ if !isOracleSelectFromDual(selectStmt) {+ return false, nil+ }+ return !hasOracleSubqueriesInSelection(selectStmt), nil+}++func isSimpleOracleSelect(selectStmt *oracleast.SelectStmt) bool {+ return selectStmt.WithClause == nil &&+ !selectStmt.Distinct &&+ !selectStmt.UniqueKw &&+ !selectStmt.All &&+ selectStmt.Into == nil &&+ selectStmt.IntoVars == nil &&+ selectStmt.WhereClause == nil &&+ selectStmt.Hierarchical == nil &&+ selectStmt.GroupClause == nil &&+ selectStmt.HavingClause == nil &&+ selectStmt.ModelClause == nil &&+ len(selectStmt.WindowDefs) == 0 &&+ selectStmt.QualifyClause == nil &&+ selectStmt.OrderBy == nil &&+ selectStmt.ForUpdate == nil &&+ selectStmt.FetchFirst == nil &&+ selectStmt.Op == oracleast.SETOP_NONE+}++func isOracleSelectFromDual(selectStmt *oracleast.SelectStmt) bool {+ if selectStmt.FromClause == nil || len(selectStmt.FromClause.Items) != 1 {+ return false+ }+ tableRef, ok := selectStmt.FromClause.Items[0].(*oracleast.TableRef)+ if !ok || tableRef.Name == nil {+ return false+ }+ return tableRef.Alias == nil &&+ tableRef.Name.Schema == "" &&+ tableRef.Name.DBLink == "" &&+ strings.EqualFold(tableRef.Name.Name, "DUAL")+}++func hasOracleSubqueriesInSelection(selectStmt *oracleast.SelectStmt) bool {+ if selectStmt.TargetList == nil || len(selectStmt.TargetList.Items) == 0 {+ return true+ }+ for _, item := range selectStmt.TargetList.Items {+ target, ok := item.(*oracleast.ResTarget)+ if !ok || target.Expr == nil {+ return true+ }+ if _, ok := target.Expr.(*oracleast.Star); ok {+ return true+ }+ hasSubquery := false+ oracleast.Inspect(target.Expr, func(node oracleast.Node) bool {+ if _, ok := node.(*oracleast.SubqueryExpr); ok {+ hasSubquery = true+ return false+ }+ return true+ })+ if hasSubquery {+ return true+ }+ }+ return false+}diff --git a/backend/plugin/db/oracle/query_test.go b/backend/plugin/parser/plsql/limit_test.gosimilarity index 97%rename from backend/plugin/db/oracle/query_test.gorename to backend/plugin/parser/plsql/limit_test.goindex c3bd7ccfe1..7d327dec36 100644--- a/backend/plugin/db/oracle/query_test.go+++ b/backend/plugin/parser/plsql/limit_test.go@@ -1,4 +1,4 @@-package oracle+package plsqlimport ("io"@@ -11,7 +11,6 @@ import ("gopkg.in/yaml.v3""github.com/bytebase/bytebase/backend/common/yamltest"- plsqlparser "github.com/bytebase/bytebase/backend/plugin/parser/plsql")func TestGetStatementWithResultLimit(t *testing.T) {@@ -51,7 +50,7 @@ func TestAddResultLimitKeepsNonSelectStatementsUnchanged(t *testing.T) {for _, tc := range tests {t.Run(tc.name, func(t *testing.T) {- got := addResultLimit(tc.statement, 10, tc.engineVersion)+ got := statementWithResultLimit(tc.statement, 10, tc.engineVersion)require.Equal(t, tc.statement, got)})@@ -83,7 +82,7 @@ func TestAddResultLimitSkipsOnlySimpleDualSelect(t *testing.T) {for _, tc := range tests {t.Run(tc.name, func(t *testing.T) {- got := addResultLimit(tc.statement, 10, "19.0.0.0.0")+ got := statementWithResultLimit(tc.statement, 10, "19.0.0.0.0")require.Equal(t, tc.want, got)})@@ -157,7 +156,7 @@ func TestAddLimitFor12cAndLaterRegressionClauseOrder(t *testing.T) {require.Equal(t, tc.want, got)assertSubstringsInOrder(t, got, tc.clausesInOrder)if tc.validateParse {- _, err := plsqlparser.ParsePLSQL(got)+ _, err := ParsePLSQL(got)require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)}})diff --git a/backend/plugin/db/oracle/test-data/test_limit.yaml b/backend/plugin/parser/plsql/test-data/test_limit.yamlsimilarity index 100%rename from backend/plugin/db/oracle/test-data/test_limit.yamlrename to backend/plugin/parser/plsql/test-data/test_limit.yamldiff --git a/backend/plugin/parser/redshift/limit.go b/backend/plugin/parser/redshift/limit.gonew file mode 100644index 0000000000..15290db615--- /dev/null+++ b/backend/plugin/parser/redshift/limit.go@@ -0,0 +1,134 @@+package redshift++import (+ "fmt"+ "log/slog"+ "strings"++ redshiftast "github.com/bytebase/omni/redshift/ast"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_REDSHIFT, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for Redshift, which+// takes no engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := statementWithResultLimitInline(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ // To handle cases where there are comments in the query.+ // eg. select * from t1 -- this is comment;+ // Add two new line symbols here.+ return fmt.Sprintf("WITH result AS (\n%s\n) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)+ }+ return stmt+}++func statementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")+ }++ stmts, err := ParseRedshift(statement)+ if err != nil {+ return "", errors.Wrap(err, "failed to parse statement")+ }+ if len(stmts) != 1 {+ return "", errors.Errorf("expected exactly one statement, got %d", len(stmts))+ }++ sel, ok := stmts[0].AST.(*redshiftast.SelectStmt)+ if !ok {+ // Non-SELECT statement, return as-is.+ return statement, nil+ }++ return rewriteSelectLimit(statement, sel, limitCount)+}++// rewriteSelectLimit adds or adjusts the LIMIT clause of a SELECT statement+// using byte-offset positions from the omni AST to surgically edit the original SQL text.+func rewriteSelectLimit(sql string, sel *redshiftast.SelectStmt, limitCount int) (string, error) {+ if sel.LimitCount != nil {+ existingLimit, isInteger := extractIntFromNode(sel.LimitCount)+ loc := nodeLocOf(sel.LimitCount)+ if isInteger && existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil // existing limit is already lower or equal, keep it+ }+ if isInteger && loc.Start >= 0 && loc.End > loc.Start && loc.End <= len(sql) {+ return sql[:loc.Start] + fmt.Sprintf("%d", limitCount) + sql[loc.End:], nil+ }+ // ALL/NULL and non-constant limits lack reliable replacement spans.+ // Cap them with the outer query instead of rewriting their text.+ return "", errors.Errorf("cannot rewrite LIMIT expression in place")+ }++ // No LIMIT clause — find the right insertion point.+ // Redshift's PostgreSQL-derived grammar order: ... ORDER BY ... LIMIT ... FOR UPDATE ...+ // LIMIT goes BEFORE FOR UPDATE but AFTER everything else.+ insertPos, beforeLocking := findLimitInsertPosition(sel)+ if beforeLocking {+ // Inserting at the start of FOR UPDATE/SHARE. The original whitespace+ // before FOR becomes the separator before LIMIT; we add a trailing+ // space to separate the limit value from FOR.+ return sql[:insertPos] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[insertPos:], nil+ }+ return sql[:insertPos] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[insertPos:], nil+}++// findLimitInsertPosition returns the byte offset where " LIMIT N" should be inserted,+// and whether the insertion is before a locking clause (FOR UPDATE/SHARE).+func findLimitInsertPosition(sel *redshiftast.SelectStmt) (int, bool) {+ // LIMIT must appear before FOR UPDATE/SHARE.+ if sel.LockingClause != nil {+ span := redshiftast.ListSpan(sel.LockingClause)+ if span.Start > 0 {+ return span.Start, true+ }+ }++ // Otherwise insert at SelectStmt.Loc.End (after everything, including outer parens).+ // For CTE queries, omni may report SelectStmt.Loc.End before the outer ORDER BY,+ // so account for the sort clause explicitly.+ end := sel.Loc.End+ if span := redshiftast.ListSpan(sel.SortClause); span.End > end {+ end = span.End+ }+ if end <= 0 {+ return 0, false+ }+ return end, false+}++// extractIntFromNode distinguishes integer limits, including zero, from unlimited+// ALL/NULL constants and expressions that require the fallback wrapper.+func extractIntFromNode(node redshiftast.Node) (int, bool) {+ switch n := node.(type) {+ case *redshiftast.Integer:+ return int(n.Ival), true+ case *redshiftast.A_Const:+ if iv, ok := n.Val.(*redshiftast.Integer); ok {+ return int(iv.Ival), true+ }+ return 0, false+ default:+ return 0, false+ }+}++// nodeLocOf returns the Loc of a node, handling common wrapper types.+func nodeLocOf(node redshiftast.Node) redshiftast.Loc {+ switch n := node.(type) {+ case *redshiftast.A_Const:+ return n.Loc+ default:+ return redshiftast.Loc{Start: -1, End: -1}+ }+}diff --git a/backend/plugin/parser/redshift/limit_test.go b/backend/plugin/parser/redshift/limit_test.gonew file mode 100644index 0000000000..ae2eabb7c9--- /dev/null+++ b/backend/plugin/parser/redshift/limit_test.go@@ -0,0 +1,228 @@+package redshift++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestGetStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ want string+ }{+ {+ name: "Simple SELECT without LIMIT",+ statement: "SELECT * FROM users",+ limit: 10,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "SELECT with existing LIMIT higher than requested",+ statement: "SELECT * FROM users LIMIT 100",+ limit: 10,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "SELECT with existing LIMIT lower than requested",+ statement: "SELECT * FROM users LIMIT 5",+ limit: 10,+ want: "SELECT * FROM users LIMIT 5",+ },+ {+ name: "SELECT with existing LIMIT 0 is kept, not replaced",+ statement: "SELECT * FROM users LIMIT 0",+ limit: 10,+ want: "SELECT * FROM users LIMIT 0",+ },+ {+ name: "WITH query (CTE)",+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users",+ limit: 10,+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users LIMIT 10",+ },+ {+ name: "SELECT with ORDER BY",+ statement: "SELECT * FROM users ORDER BY created_at DESC",+ limit: 10,+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",+ },+ {+ name: "UNION query",+ statement: "SELECT id FROM users UNION SELECT id FROM admins",+ limit: 10,+ want: "SELECT id FROM users UNION SELECT id FROM admins LIMIT 10",+ },+ {+ name: "SELECT with FOR UPDATE",+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",+ },+ {+ name: "Non-SELECT statement (INSERT)",+ statement: "INSERT INTO users (name) VALUES ('John')",+ limit: 10,+ want: "INSERT INTO users (name) VALUES ('John')",+ },+ {+ name: "Non-SELECT statement (DELETE)",+ statement: "DELETE FROM users WHERE active = false",+ limit: 10,+ want: "DELETE FROM users WHERE active = false",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := statementWithResultLimit(tt.statement, tt.limit, "")++ if tt.statement == tt.want {+ assert.Equal(t, tt.want, got)+ } else {+ assert.Contains(t, got, "LIMIT")+ }+ })+ }+}++func TestGetStatementWithResultLimitInline(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ wantErr bool+ want string+ }{+ {+ name: "Valid SELECT statement",+ statement: "SELECT * FROM users",+ limit: 10,+ want: "SELECT * FROM users LIMIT 10",+ },+ {+ name: "Invalid SQL syntax",+ statement: "SELECT * FROM WHERE",+ limit: 10,+ wantErr: true,+ },+ {+ name: "Empty statement",+ statement: "",+ limit: 10,+ wantErr: true,+ },+ {+ name: "FOR UPDATE gets proper whitespace",+ statement: "SELECT * FROM users WHERE id = 1 FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users WHERE id = 1 LIMIT 10 FOR UPDATE",+ },+ {+ name: "Non-constant LIMIT expression falls back to error",+ statement: "SELECT * FROM t LIMIT (1+2)",+ limit: 5,+ wantErr: true,+ },+ {+ name: "LIMIT 0 is kept, not replaced with the requested cap",+ statement: "SELECT * FROM t LIMIT 0",+ limit: 5,+ want: "SELECT * FROM t LIMIT 0",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)+ if tt.wantErr {+ require.Error(t, err)+ return+ }+ require.NoError(t, err)+ assert.NotEmpty(t, got)+ if tt.want != "" {+ assert.Equal(t, tt.want, got)+ }+ })+ }+}++// TestGetStatementWithResultLimitFallback locks Redshift's own fallback text,+// distinct from PostgreSQL's: a blind CTE-wrap for a statement the AST rewrite+// cannot handle, still never applied to a statement that already parsed clean.+func TestGetStatementWithResultLimitFallback(t *testing.T) {+ got := statementWithResultLimit("SELECT * FROM t LIMIT (1+2)", 5, "")+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT (1+2)\n) SELECT * FROM result LIMIT 5;", got)+}++func TestGetStatementWithResultLimitInlineClauseOrder(t *testing.T) {+ tests := []struct {+ name string+ statement string+ limit int+ want string+ clausesInOrder []string+ }{+ {+ name: "ORDER BY",+ statement: "SELECT * FROM users ORDER BY created_at DESC",+ limit: 10,+ want: "SELECT * FROM users ORDER BY created_at DESC LIMIT 10",+ clausesInOrder: []string{"ORDER BY", "LIMIT 10"},+ },+ {+ name: "ORDER BY with FOR UPDATE",+ statement: "SELECT * FROM users ORDER BY id FOR UPDATE",+ limit: 10,+ want: "SELECT * FROM users ORDER BY id LIMIT 10 FOR UPDATE",+ clausesInOrder: []string{"ORDER BY", "LIMIT 10", "FOR UPDATE"},+ },+ {+ name: "WITH query with outer ORDER BY",+ statement: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at",+ limit: 10,+ want: "WITH active_users AS (SELECT * FROM users WHERE active = true) SELECT * FROM active_users ORDER BY created_at LIMIT 10",+ clausesInOrder: []string{"WITH", "ORDER BY", "LIMIT 10"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := statementWithResultLimitInline(tt.statement, tt.limit)+ require.NoError(t, err)+ if tt.want != "" {+ assert.Equal(t, tt.want, got)+ }+ assertSubstringsInOrder(t, got, tt.clausesInOrder)++ _, err = ParseRedshift(got)+ require.NoError(t, err, "rewritten SQL should remain parseable: %s", got)+ })+ }+}++func assertSubstringsInOrder(t *testing.T, s string, substrings []string) {+ t.Helper()++ start := 0+ for _, substring := range substrings {+ index := strings.Index(s[start:], substring)+ require.NotEqualf(t, -1, index, "expected %q after offset %d in %q", substring, start, s)+ start += index + len(substring)+ }+}++func TestUnlimitedResultLimit(t *testing.T) {+ for _, clause := range []string{"ALL", "NULL"} {+ t.Run(clause, func(t *testing.T) {+ statement := "SELECT * FROM t LIMIT " + clause + " OFFSET 2;"+ got := statementWithResultLimit(statement, 10, "")+ require.Equal(t, "WITH result AS (\nSELECT * FROM t LIMIT "+clause+" OFFSET 2\n) SELECT * FROM result LIMIT 10;", got)+ })+ }+}diff --git a/backend/plugin/db/snowflake/query.go b/backend/plugin/parser/snowflake/limit.gosimilarity index 83%rename from backend/plugin/db/snowflake/query.gorename to backend/plugin/parser/snowflake/limit.goindex 2d32289c93..609ae35d2d 100644--- a/backend/plugin/db/snowflake/query.go+++ b/backend/plugin/parser/snowflake/limit.go@@ -11,22 +11,27 @@ import ("github.com/pkg/errors""github.com/bytebase/bytebase/backend/common/log"- "github.com/bytebase/bytebase/backend/plugin/db/util"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/utils")-const stmtErrFmt = "statement: %s"+func init() {+ base.RegisterResultLimitFunc(storepb.Engine_SNOWFLAKE, statementWithResultLimit)+}-func getStatementWithResultLimit(statement string, limit int) string {- stmt, err := getStatementWithResultLimitInline(statement, limit)+// statementWithResultLimit implements base.ResultLimitFunc for Snowflake, which+// takes no engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := statementWithResultLimitInline(statement, limit)if err != nil {slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))- return fmt.Sprintf("SELECT * FROM (%s) LIMIT %d", util.TrimStatement(statement), limit)+ return fmt.Sprintf("SELECT * FROM (%s) LIMIT %d", base.TrimStatement(statement), limit)}return stmt}-// getStatementWithResultLimitInline rewrites the single statement so that its+// statementWithResultLimitInline rewrites the single statement so that its// outermost query returns at most limitCount rows. The rewrite splices the// replacement into the original statement text via omni AST byte offsets// (ast.Loc), preserving the original formatting. The output shape mirrors the@@ -44,12 +49,12 @@ func getStatementWithResultLimit(statement string, limit int) string {// Non-query statements are returned unchanged (the legacy listener never// fired on them). Any error makes the caller fall back to wrapping the// statement in SELECT * FROM (...) LIMIT n.-func getStatementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {+func statementWithResultLimitInline(singleStatement string, limitCount int) (string, error) {trimmed := strings.TrimRightFunc(singleStatement, utils.IsSpaceOrSemicolon)file, err := omniparser.Parse(trimmed)if err != nil {- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)+ return "", errors.Wrapf(err, "statement: %s", singleStatement)}if len(file.Stmts) != 1 {return "", errors.Errorf("expected exactly 1 statement, got %d", len(file.Stmts))@@ -65,7 +70,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (if sel.Limit != nil {spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Limit), limitCount)if err != nil {- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)+ return "", errors.Wrapf(err, "statement: %s", singleStatement)}return spliced + "\n;", nil}@@ -74,7 +79,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (if sel.Fetch != nil {spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Fetch.Count), limitCount)if err != nil {- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)+ return "", errors.Wrapf(err, "statement: %s", singleStatement)}return spliced + "\n;", nil}@@ -90,7 +95,7 @@ func getStatementWithResultLimitInline(singleStatement string, limitCount int) (if sel.Top != nil {spliced, err := spliceLimitNumber(trimmed, omniast.NodeLoc(sel.Top), limitCount)if err != nil {- return "", errors.Wrapf(err, stmtErrFmt, singleStatement)+ return "", errors.Wrapf(err, "statement: %s", singleStatement)}return spliced + "\n;", nil}diff --git a/backend/plugin/db/snowflake/query_test.go b/backend/plugin/parser/snowflake/limit_test.gosimilarity index 94%rename from backend/plugin/db/snowflake/query_test.gorename to backend/plugin/parser/snowflake/limit_test.goindex 680c8ccee4..55ab8de248 100644--- a/backend/plugin/db/snowflake/query_test.go+++ b/backend/plugin/parser/snowflake/limit_test.go@@ -35,7 +35,7 @@ func runLimitTest(t *testing.T, file string, record bool) {require.NoError(t, err)for i, tc := range testCases {- want := getStatementWithResultLimit(tc.Stmt, tc.Limit)+ want := statementWithResultLimit(tc.Stmt, tc.Limit, "")if record {testCases[i].Want = want} else {diff --git a/backend/plugin/db/snowflake/test-data/test_limit.yaml b/backend/plugin/parser/snowflake/test-data/test_limit.yamlsimilarity index 100%rename from backend/plugin/db/snowflake/test-data/test_limit.yamlrename to backend/plugin/parser/snowflake/test-data/test_limit.yamldiff --git a/backend/plugin/parser/spanner/limit.go b/backend/plugin/parser/spanner/limit.gonew file mode 100644index 0000000000..7d15ba8172--- /dev/null+++ b/backend/plugin/parser/spanner/limit.go@@ -0,0 +1,42 @@+package spanner++import (+ "strconv"++ "github.com/bytebase/omni/googlesql/ast"+ "github.com/bytebase/omni/googlesql/parser"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_SPANNER, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for Spanner, which+// takes no engineVersion. It sets the LIMIT of the outermost query in stmt to at+// most limit. It never wraps stmt in a subquery, where Spanner rejects a WITH+// clause, so a query it cannot rewrite runs unchanged and queryStatement caps its+// rows.+func statementWithResultLimit(stmt string, limit int, _ string) string {+ file, errs := parser.Parse(stmt)+ if len(errs) > 0 || len(file.Stmts) != 1 {+ return stmt+ }+ query, ok := file.Stmts[0].(*ast.QueryStmt)+ // FOR UPDATE fails in queryStatement's read-only transaction, so there is no+ // point fitting a LIMIT before it.+ if !ok || query.ForUpdate {+ return stmt+ }+ if query.Limit == nil {+ end := query.Loc.End+ return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]+ }+ count, ok := query.Limit.(*ast.Literal)+ if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {+ return stmt+ }+ return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]+}diff --git a/backend/plugin/parser/spanner/limit_test.go b/backend/plugin/parser/spanner/limit_test.gonew file mode 100644index 0000000000..6d15c84587--- /dev/null+++ b/backend/plugin/parser/spanner/limit_test.go@@ -0,0 +1,81 @@+package spanner++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want string+ }{+ {+ name: "select",+ statement: "SELECT * FROM t",+ want: "SELECT * FROM t LIMIT 100",+ },+ {+ name: "with",+ statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",+ want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",+ },+ {+ name: "lowercase with, order by, and trailing semicolon",+ statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",+ want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",+ },+ {+ name: "with and set operation",+ statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",+ want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",+ },+ {+ name: "set operation",+ statement: "SELECT 1 UNION ALL SELECT 2",+ want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",+ },+ {+ name: "inner limit belongs to the parenthesized query",+ statement: "(SELECT x FROM t LIMIT 500)",+ want: "(SELECT x FROM t LIMIT 500) LIMIT 100",+ },+ {+ name: "trailing comment",+ statement: "SELECT x FROM t -- note",+ want: "SELECT x FROM t LIMIT 100 -- note",+ },+ {+ name: "lower limit is kept",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",+ },+ {+ name: "higher limit is lowered",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",+ },+ {+ name: "non-literal limit is unchanged",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",+ },+ {+ name: "for update is unchanged",+ statement: "SELECT x FROM t ORDER BY x FOR UPDATE",+ want: "SELECT x FROM t ORDER BY x FOR UPDATE",+ },+ {+ name: "unparsable query is unchanged",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, statementWithResultLimit(tc.statement, 100, ""))+ })+ }+}diff --git a/backend/plugin/parser/standard/limit.go b/backend/plugin/parser/standard/limit.gonew file mode 100644index 0000000000..5de8c86391--- /dev/null+++ b/backend/plugin/parser/standard/limit.go@@ -0,0 +1,20 @@+package standard++import (+ "fmt"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_CLICKHOUSE, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for ClickHouse,+// which takes no engineVersion. ClickHouse is parsed only as text here (see+// query.go), so there is no AST to place the LIMIT precisely; every statement+// is wrapped in a CTE.+func statementWithResultLimit(statement string, limit int, _ string) string {+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)+}diff --git a/backend/plugin/parser/standard/limit_test.go b/backend/plugin/parser/standard/limit_test.gonew file mode 100644index 0000000000..c5a7678a0b--- /dev/null+++ b/backend/plugin/parser/standard/limit_test.go@@ -0,0 +1,23 @@+package standard++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ statement string+ limit int+ want string+ }{+ {"SELECT * FROM t", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},+ {"SELECT * FROM t;", 10, "WITH result AS (SELECT * FROM t) SELECT * FROM result LIMIT 10;"},+ {"DELETE FROM t", 5, "WITH result AS (DELETE FROM t) SELECT * FROM result LIMIT 5;"},+ }+ for _, tc := range tests {+ got := statementWithResultLimit(tc.statement, tc.limit, "")+ require.Equalf(t, tc.want, got, "%q", tc.statement)+ }+}diff --git a/backend/plugin/parser/starrocks/limit.go b/backend/plugin/parser/starrocks/limit.gonew file mode 100644index 0000000000..80f3ac49f8--- /dev/null+++ b/backend/plugin/parser/starrocks/limit.go@@ -0,0 +1,227 @@+package starrocks++import (+ "fmt"+ "log/slog"+ "strconv"+ "strings"++ "github.com/bytebase/omni/starrocks/ast"+ "github.com/bytebase/omni/starrocks/parser"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_STARROCKS, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for StarRocks, which+// takes no engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ trimmedStatement := strings.TrimSpace(statement)+ if strings.HasPrefix(strings.ToUpper(trimmedStatement), "SHOW") {+ return statement+ }++ stmt, err := statementWithResultLimitInline(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ return fmt.Sprintf("SELECT * FROM (%s) result LIMIT %d;", base.TrimStatement(statement), limit)+ }+ return stmt+}++func statementWithResultLimitInline(statement string, limitCount int) (string, error) {+ if strings.TrimSpace(statement) == "" {+ return "", errors.New("empty statement")+ }++ file, errs := parser.Parse(statement)+ if len(errs) > 0 {+ return "", errors.New(errs[0].Error())+ }+ if file == nil || len(file.Stmts) != 1 {+ stmtCount := 0+ if file != nil {+ stmtCount = len(file.Stmts)+ }+ return "", errors.Errorf("expected exactly one statement, got %d", stmtCount)+ }++ switch stmt := file.Stmts[0].(type) {+ case *ast.SelectStmt:+ return rewriteSelectLimit(statement, stmt, limitCount)+ case *ast.SetOpStmt:+ return rewriteSetOpLimit(statement, stmt, limitCount)+ case *ast.ParenSelect:+ // The wrapper itself may carry the trailing clauses —+ // (SELECT 1) LIMIT 900000 attaches the LIMIT to the ParenSelect,+ // not the inner query — so the wrapper is where the cap applies.+ if stmt.Limit != nil {+ return rewriteExistingLimit(statement, stmt.Limit, limitCount)+ }+ // No wrapper LIMIT: append one at the wrapper level, which covers+ // any paren nesting depth — ((SELECT 1)) LIMIT n and+ // (SELECT 1) ORDER BY 1 LIMIT n are engine-valid — instead of+ // unwrapping and missing deeper layers.+ if hasUnparsedTail(statement, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")+ }+ return statement[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + statement[stmt.Loc.End:], nil+ default:+ return statement, nil+ }+}++func hasUnparsedTail(sql string, locEnd int) bool {+ pos := locEnd+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ for pos < len(sql) && sql[pos] == ';' {+ pos+++ }+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if pos >= len(sql) {+ return false+ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return false+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return false+ }+ pos += end + 2+ continue+ }+ return true+ }+ return false+}++// rewriteExistingLimit caps an already-present LIMIT clause: keep it when it+// is a constant no larger than limitCount, replace the count otherwise.+func rewriteExistingLimit(sql string, limitNode ast.Node, limitCount int) (string, error) {+ limitLoc := literalLoc(limitNode)+ if limitLoc.Start < 0 {+ return "", errors.New("cannot rewrite non-constant LIMIT expression")+ }++ if countStart, countEnd, ok := findCommaLimitCount(sql, limitLoc); ok {+ // A literal 0 count (LIMIT offset,0) is a valid, stricter limit and must+ // be kept, matching the non-comma path below.+ existingCount, _ := strconv.Atoi(sql[countStart:countEnd])+ if existingCount >= 0 && existingCount <= limitCount {+ return sql, nil+ }+ return sql[:countStart] + fmt.Sprintf("%d", limitCount) + sql[countEnd:], nil+ }++ existingLimit := extractLimitValue(limitNode)+ if existingLimit >= 0 && existingLimit <= limitCount {+ return sql, nil+ }+ return sql[:limitLoc.Start] + fmt.Sprintf("%d", limitCount) + sql[limitLoc.End:], nil+}++func rewriteSelectLimit(sql string, stmt *ast.SelectStmt, limitCount int) (string, error) {+ if stmt.Limit != nil {+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)+ }++ if stmt.Into != nil {+ return sql[:stmt.Into.Loc.Start] + fmt.Sprintf("LIMIT %d ", limitCount) + sql[stmt.Into.Loc.Start:], nil+ }+ if hasUnparsedTail(sql, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")+ }+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil+}++func rewriteSetOpLimit(sql string, stmt *ast.SetOpStmt, limitCount int) (string, error) {+ if stmt.Limit != nil {+ return rewriteExistingLimit(sql, stmt.Limit, limitCount)+ }+ if hasUnparsedTail(sql, stmt.Loc.End) {+ return "", errors.New("statement has unparsed tail content")+ }+ return sql[:stmt.Loc.End] + fmt.Sprintf(" LIMIT %d", limitCount) + sql[stmt.Loc.End:], nil+}++func extractLimitValue(node ast.Node) int {+ lit, ok := node.(*ast.Literal)+ if !ok || lit.Kind != ast.LitInt {+ return -1+ }+ val, err := strconv.Atoi(lit.Value)+ if err != nil {+ return -1+ }+ return val+}++func literalLoc(node ast.Node) ast.Loc {+ lit, ok := node.(*ast.Literal)+ if !ok {+ return ast.Loc{Start: -1, End: -1}+ }+ return lit.Loc+}++func findCommaLimitCount(sql string, limitLoc ast.Loc) (countStart, countEnd int, found bool) {+ pos := limitLoc.End+ pos = skipWhitespaceAndComments(sql, pos)+ if pos >= len(sql) || sql[pos] != ',' {+ return 0, 0, false+ }+ pos+++ pos = skipWhitespaceAndComments(sql, pos)+ countStart = pos+ for pos < len(sql) && sql[pos] >= '0' && sql[pos] <= '9' {+ pos+++ }+ if pos == countStart {+ return 0, 0, false+ }+ return countStart, pos, true+}++func skipWhitespaceAndComments(sql string, pos int) int {+ for pos < len(sql) {+ for pos < len(sql) && (sql[pos] == ' ' || sql[pos] == '\t' || sql[pos] == '\n' || sql[pos] == '\r') {+ pos+++ }+ if strings.HasPrefix(sql[pos:], "--") || strings.HasPrefix(sql[pos:], "#") {+ nl := strings.IndexByte(sql[pos:], '\n')+ if nl < 0 {+ return len(sql)+ }+ pos += nl + 1+ continue+ }+ if strings.HasPrefix(sql[pos:], "/*") {+ end := strings.Index(sql[pos:], "*/")+ if end < 0 {+ return len(sql)+ }+ pos += end + 2+ continue+ }+ return pos+ }+ return pos+}diff --git a/backend/plugin/db/starrocks/query_test.go b/backend/plugin/parser/starrocks/limit_test.gosimilarity index 97%rename from backend/plugin/db/starrocks/query_test.gorename to backend/plugin/parser/starrocks/limit_test.goindex c50d65dda4..6ffec593af 100644--- a/backend/plugin/db/starrocks/query_test.go+++ b/backend/plugin/parser/starrocks/limit_test.go@@ -94,6 +94,12 @@ func TestGetStatementWithResultLimit(t *testing.T) {count: 10,want: "SELECT * FROM t LIMIT 123,10;",},+ {+ // A literal 0 count is kept, not replaced with the requested cap.+ stmt: "SELECT * FROM t LIMIT 5,0;",+ count: 10,+ want: "SELECT * FROM t LIMIT 5,0;",+ },{stmt: "SELECT * FROM t LIMIT 0,\n1000000;",count: 10,@@ -264,7 +270,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {}for _, tc := range testCases {- got := getStatementWithResultLimit(tc.stmt, tc.count)+ got := statementWithResultLimit(tc.stmt, tc.count, "")require.Equal(t, tc.want, got, tc.stmt)}}diff --git a/backend/plugin/parser/tidb/limit.go b/backend/plugin/parser/tidb/limit.gonew file mode 100644index 0000000000..f4fcbfc1c5--- /dev/null+++ b/backend/plugin/parser/tidb/limit.go@@ -0,0 +1,87 @@+package tidb++import (+ "fmt"+ "log/slog"+ "strings"++ tidbast "github.com/pingcap/tidb/pkg/parser/ast"+ "github.com/pingcap/tidb/pkg/parser/format"+ tidbdriver "github.com/pingcap/tidb/pkg/types/parser_driver"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_TIDB, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for TiDB, which+// takes no engineVersion.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := statementWithResultLimitInline(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result LIMIT %d;", base.TrimStatement(statement), limit)+ }+ return stmt+}++func statementWithResultLimitInline(statement string, limit int) (string, error) {+ stmtList, err := ParseTiDB(statement, "", "")+ if err != nil {+ return "", errors.Wrapf(err, "failed to parse tidb statement: %s", statement)+ }+ if len(stmtList) != 1 {+ return "", errors.Errorf("expect one single statement in the query, %s", statement)+ }+ restoreFlags := format.DefaultRestoreFlags | format.RestoreStringWithoutDefaultCharset+ stmt := stmtList[0]+ switch stmt := stmt.(type) {+ case *tidbast.SelectStmt:+ if stmt.Limit != nil && stmt.Limit.Count != nil {+ if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {+ userLimit := int(v.GetInt64())+ if limit < userLimit {+ userLimit = limit+ }+ stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")+ }+ } else {+ stmt.Limit = &tidbast.Limit{+ Count: tidbast.NewValueExpr(int64(limit), "", ""),+ }+ }+ var buffer strings.Builder+ ctx := format.NewRestoreCtx(restoreFlags, &buffer)+ if err := stmt.Restore(ctx); err != nil {+ return "", err+ }+ return buffer.String(), nil+ case *tidbast.SetOprStmt:+ if stmt.Limit != nil && stmt.Limit.Count != nil {+ if v, ok := stmt.Limit.Count.(*tidbdriver.ValueExpr); ok {+ userLimit := int(v.GetInt64())+ if limit < userLimit {+ userLimit = limit+ }+ stmt.Limit.Count = tidbast.NewValueExpr(int64(userLimit), "", "")+ }+ } else {+ stmt.Limit = &tidbast.Limit{+ Count: tidbast.NewValueExpr(int64(limit), "", ""),+ }+ }+ var buffer strings.Builder+ ctx := format.NewRestoreCtx(restoreFlags, &buffer)+ if err := stmt.Restore(ctx); err != nil {+ return "", err+ }+ return buffer.String(), nil+ default:+ }+ return statement, nil+}diff --git a/backend/plugin/db/tidb/query_test.go b/backend/plugin/parser/tidb/limit_test.gosimilarity index 95%rename from backend/plugin/db/tidb/query_test.gorename to backend/plugin/parser/tidb/limit_test.goindex b80f5a232b..bebf7ecc55 100644--- a/backend/plugin/db/tidb/query_test.go+++ b/backend/plugin/parser/tidb/limit_test.go@@ -55,7 +55,7 @@ func TestGetStatementWithResultLimit(t *testing.T) {}for _, tc := range testCases {- got := getStatementWithResultLimit(tc.stmt, tc.count)+ got := statementWithResultLimit(tc.stmt, tc.count, "")require.Equal(t, tc.want, got, tc.stmt)}}diff --git a/backend/plugin/parser/tsql/limit.go b/backend/plugin/parser/tsql/limit.gonew file mode 100644index 0000000000..a9d99d560b--- /dev/null+++ b/backend/plugin/parser/tsql/limit.go@@ -0,0 +1,28 @@+package tsql++import (+ "fmt"+ "log/slog"++ omnimssql "github.com/bytebase/omni/mssql"++ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterResultLimitFunc(storepb.Engine_MSSQL, statementWithResultLimit)+}++// statementWithResultLimit implements base.ResultLimitFunc for SQL Server,+// which takes no engineVersion. The rewrite itself lives in omni, which already+// owns the T-SQL grammar this needs.+func statementWithResultLimit(statement string, limit int, _ string) string {+ stmt, err := omnimssql.StatementWithResultLimit(statement, limit)+ if err != nil {+ slog.Error("fail to add limit clause", slog.String("statement", statement), log.BBError(err))+ return fmt.Sprintf("WITH result AS (%s) SELECT TOP %d * FROM result;", base.TrimStatement(statement), limit)+ }+ return stmt+}diff --git a/backend/plugin/db/mssql/query_test.go b/backend/plugin/parser/tsql/limit_test.gosimilarity index 93%rename from backend/plugin/db/mssql/query_test.gorename to backend/plugin/parser/tsql/limit_test.goindex de93735150..aa0dc2944f 100644--- a/backend/plugin/db/mssql/query_test.go+++ b/backend/plugin/parser/tsql/limit_test.go@@ -1,4 +1,4 @@-package mssql+package tsqlimport ("io"@@ -35,7 +35,7 @@ func runLimitTest(t *testing.T, file string, record bool) {require.NoError(t, err)for i, tc := range testCases {- want := getStatementWithResultLimit(tc.Stmt, tc.Limit)+ want := statementWithResultLimit(tc.Stmt, tc.Limit, "")if record {testCases[i].Want = want} else {diff --git a/backend/plugin/db/mssql/test-data/test_limit.yaml b/backend/plugin/parser/tsql/test-data/test_limit.yamlsimilarity index 100%rename from backend/plugin/db/mssql/test-data/test_limit.yamlrename to backend/plugin/parser/tsql/test-data/test_limit.yaml