Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(spanner): run queries that start with WITH when a row limit applies (#21478)

Danny Xu13228ad54f7c
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.go
index c1cf1849bf..cd85749c63 100644
--- a/backend/plugin/db/spanner/spanner.go
+++ b/backend/plugin/db/spanner/spanner.go
@@ -18,6 +18,8 @@ import (
spannerdb "cloud.google.com/go/spanner/admin/database/apiv1"
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
+ "github.com/bytebase/omni/googlesql/ast"
+ "github.com/bytebase/omni/googlesql/parser"
"github.com/pkg/errors"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
@@ -33,7 +35,6 @@ import (
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/utils"
)
var (
@@ -303,17 +304,30 @@ func getColumnTypeName(columnType *sppb.Type) (string, error) {
return columnType.Code.String(), nil
}
-// getStatementWithResultLimit wraps a SQL statement in a CTE to enforce a result limit.
-// This is a simple approach that works for SELECT queries but has a critical limitation:
-// Spanner does NOT support DML statements (INSERT/UPDATE/DELETE) inside CTEs.
-//
-// This function should ONLY be called for SELECT statements (verify with util.IsSelect first).
-// For a more robust parser-based approach that can handle complex queries, see the
-// PostgreSQL/MySQL implementations which parse and inject LIMIT clauses directly.
+// getStatementWithResultLimit sets the LIMIT of the outermost query in stmt to
+// at most limit. It never wraps stmt in a subquery, where Spanner rejects a
+// WITH clause, so a query it cannot rewrite runs unchanged and queryStatement
+// caps its rows.
func getStatementWithResultLimit(stmt string, limit int) string {
- stmt = strings.TrimRightFunc(stmt, utils.IsSpaceOrSemicolon)
- limitPart := fmt.Sprintf(" LIMIT %d", limit)
- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", stmt, limitPart)
+ file, errs := parser.Parse(stmt)
+ if len(errs) > 0 || len(file.Stmts) != 1 {
+ return stmt
+ }
+ query, ok := file.Stmts[0].(*ast.QueryStmt)
+ // FOR UPDATE fails in queryStatement's read-only transaction, so there is no
+ // point fitting a LIMIT before it.
+ if !ok || query.ForUpdate {
+ return stmt
+ }
+ if query.Limit == nil {
+ end := query.Loc.End
+ return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]
+ }
+ count, ok := query.Limit.(*ast.Literal)
+ if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {
+ return stmt
+ }
+ return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]
}
// instancePath returns the Spanner instance resource name,
@@ -357,7 +371,6 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
startTime := time.Now()
queryResult, err := func() (*v1pb.QueryResult, error) {
if util.IsSelect(statement) {
- // Only apply limit wrapper for SELECT statements
if queryContext.Limit > 0 {
statement = getStatementWithResultLimit(statement, queryContext.Limit)
}
@@ -448,6 +461,9 @@ func (d *Driver) queryStatement(ctx context.Context, statement string, queryCont
result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
break
}
+ if queryContext.Limit > 0 && n >= queryContext.Limit {
+ break
+ }
row, err = iter.Next()
if err == iterator.Done {
diff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.go
index b44750fb8c..1e378453ba 100644
--- a/backend/plugin/db/spanner/spanner_test.go
+++ b/backend/plugin/db/spanner/spanner_test.go
@@ -17,19 +17,27 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
)
-// TestExplainDMLWithoutRunningIt needs a Spanner that plans queries, which CI
-// cannot provide and the emulator does not do. Spanner Omni works: the client
-// reaches it without TLS or credentials through SPANNER_EMULATOR_HOST, and
-// SPANNER_TEST_DATABASE names a database in it as
-// projects/default/instances/default/databases/d.
-func TestExplainDMLWithoutRunningIt(t *testing.T) {
+// testDatabase returns the database that SPANNER_TEST_DATABASE names, as
+// projects/default/instances/default/databases/d, in the Spanner that the client
+// reaches without TLS or credentials through SPANNER_EMULATOR_HOST. CI provides
+// no Spanner, so the test is skipped without them.
+func testDatabase(t *testing.T) string {
+ t.Helper()
database := os.Getenv("SPANNER_TEST_DATABASE")
if os.Getenv("SPANNER_EMULATOR_HOST") == "" || database == "" {
- t.Skip("set SPANNER_EMULATOR_HOST and SPANNER_TEST_DATABASE to run against Spanner Omni")
+ t.Skip("set SPANNER_EMULATOR_HOST and SPANNER_TEST_DATABASE to run against Spanner")
}
+ return database
+}
+
+// TestExplainDMLWithoutRunningIt needs a Spanner that plans queries, which the
+// emulator does not do and Spanner Omni does.
+func TestExplainDMLWithoutRunningIt(t *testing.T) {
+ database := testDatabase(t)
a := require.New(t)
ctx := context.Background()
@@ -74,6 +82,108 @@ func TestExplainDMLWithoutRunningIt(t *testing.T) {
a.Equal(int64(10), total)
}
+func TestQueryConnLimitsQueryStartingWithWith(t *testing.T) {
+ a := require.New(t)
+ ctx := context.Background()
+ client, err := spanner.NewClient(ctx, testDatabase(t))
+ a.NoError(err)
+ defer client.Close()
+
+ driver := &Driver{client: client}
+ for _, statement := range []string{
+ "WITH n AS (SELECT x FROM UNNEST([3, 1, 2]) AS x) SELECT x FROM n ORDER BY x",
+ // The rewrite keeps a non-literal LIMIT, so the driver caps the rows.
+ "WITH n AS (SELECT x FROM UNNEST([3, 1, 2]) AS x) SELECT x FROM n ORDER BY x LIMIT CAST(3 AS INT64)",
+ } {
+ results, err := driver.QueryConn(ctx, nil, statement, db.QueryContext{
+ Limit: 2,
+ MaximumSQLResultSize: math.MaxInt64,
+ })
+ a.NoError(err)
+ a.Len(results, 1)
+ a.Empty(results[0].Error, results[0].Statement)
+ var values []int64
+ for _, row := range results[0].Rows {
+ values = append(values, row.Values[0].GetInt64Value())
+ }
+ a.Equal([]int64{1, 2}, values, results[0].Statement)
+ }
+}
+
+func TestGetStatementWithResultLimit(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want string
+ }{
+ {
+ name: "select",
+ statement: "SELECT * FROM t",
+ want: "SELECT * FROM t LIMIT 100",
+ },
+ {
+ name: "with",
+ statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",
+ want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",
+ },
+ {
+ name: "lowercase with, order by, and trailing semicolon",
+ statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",
+ want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",
+ },
+ {
+ name: "with and set operation",
+ statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",
+ want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",
+ },
+ {
+ name: "set operation",
+ statement: "SELECT 1 UNION ALL SELECT 2",
+ want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",
+ },
+ {
+ name: "inner limit belongs to the parenthesized query",
+ statement: "(SELECT x FROM t LIMIT 500)",
+ want: "(SELECT x FROM t LIMIT 500) LIMIT 100",
+ },
+ {
+ name: "trailing comment",
+ statement: "SELECT x FROM t -- note",
+ want: "SELECT x FROM t LIMIT 100 -- note",
+ },
+ {
+ name: "lower limit is kept",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",
+ },
+ {
+ name: "higher limit is lowered",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",
+ },
+ {
+ name: "non-literal limit is unchanged",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",
+ },
+ {
+ name: "for update is unchanged",
+ statement: "SELECT x FROM t ORDER BY x FOR UPDATE",
+ want: "SELECT x FROM t ORDER BY x FOR UPDATE",
+ },
+ {
+ name: "unparsable query is unchanged",
+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ require.Equal(t, tc.want, getStatementWithResultLimit(tc.statement, 100))
+ })
+ }
+}
+
func TestGetDatabaseFromDSN(t *testing.T) {
tests := []struct {
dsn string