Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(spanner): run queries that start with WITH when a row limit applies (#21478)
Danny Xu13228ad54f7c
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.goindex c1cf1849bf..cd85749c63 100644--- a/backend/plugin/db/spanner/spanner.go+++ b/backend/plugin/db/spanner/spanner.go@@ -18,6 +18,8 @@ import (spannerdb "cloud.google.com/go/spanner/admin/database/apiv1""cloud.google.com/go/spanner/admin/database/apiv1/databasepb"sppb "cloud.google.com/go/spanner/apiv1/spannerpb"+ "github.com/bytebase/omni/googlesql/ast"+ "github.com/bytebase/omni/googlesql/parser""github.com/pkg/errors""google.golang.org/api/iterator""google.golang.org/api/option"@@ -33,7 +35,6 @@ import ("github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util""github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/utils")var (@@ -303,17 +304,30 @@ func getColumnTypeName(columnType *sppb.Type) (string, error) {return columnType.Code.String(), nil}-// getStatementWithResultLimit wraps a SQL statement in a CTE to enforce a result limit.-// This is a simple approach that works for SELECT queries but has a critical limitation:-// Spanner does NOT support DML statements (INSERT/UPDATE/DELETE) inside CTEs.-//-// This function should ONLY be called for SELECT statements (verify with util.IsSelect first).-// For a more robust parser-based approach that can handle complex queries, see the-// PostgreSQL/MySQL implementations which parse and inject LIMIT clauses directly.+// getStatementWithResultLimit sets the LIMIT of the outermost query in stmt to+// at most limit. It never wraps stmt in a subquery, where Spanner rejects a+// WITH clause, so a query it cannot rewrite runs unchanged and queryStatement+// caps its rows.func getStatementWithResultLimit(stmt string, limit int) string {- stmt = strings.TrimRightFunc(stmt, utils.IsSpaceOrSemicolon)- limitPart := fmt.Sprintf(" LIMIT %d", limit)- return fmt.Sprintf("WITH result AS (%s) SELECT * FROM result%s;", stmt, limitPart)+ file, errs := parser.Parse(stmt)+ if len(errs) > 0 || len(file.Stmts) != 1 {+ return stmt+ }+ query, ok := file.Stmts[0].(*ast.QueryStmt)+ // FOR UPDATE fails in queryStatement's read-only transaction, so there is no+ // point fitting a LIMIT before it.+ if !ok || query.ForUpdate {+ return stmt+ }+ if query.Limit == nil {+ end := query.Loc.End+ return stmt[:end] + " LIMIT " + strconv.Itoa(limit) + stmt[end:]+ }+ count, ok := query.Limit.(*ast.Literal)+ if !ok || count.Kind != ast.LitInt || count.Ival <= int64(limit) {+ return stmt+ }+ return stmt[:count.Loc.Start] + strconv.Itoa(limit) + stmt[count.Loc.End:]}// instancePath returns the Spanner instance resource name,@@ -357,7 +371,6 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, qstartTime := time.Now()queryResult, err := func() (*v1pb.QueryResult, error) {if util.IsSelect(statement) {- // Only apply limit wrapper for SELECT statementsif queryContext.Limit > 0 {statement = getStatementWithResultLimit(statement, queryContext.Limit)}@@ -448,6 +461,9 @@ func (d *Driver) queryStatement(ctx context.Context, statement string, queryContresult.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)break}+ if queryContext.Limit > 0 && n >= queryContext.Limit {+ break+ }row, err = iter.Next()if err == iterator.Done {diff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.goindex b44750fb8c..1e378453ba 100644--- a/backend/plugin/db/spanner/spanner_test.go+++ b/backend/plugin/db/spanner/spanner_test.go@@ -17,19 +17,27 @@ import ("google.golang.org/protobuf/types/known/timestamppb"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util")-// TestExplainDMLWithoutRunningIt needs a Spanner that plans queries, which CI-// cannot provide and the emulator does not do. Spanner Omni works: the client-// reaches it without TLS or credentials through SPANNER_EMULATOR_HOST, and-// SPANNER_TEST_DATABASE names a database in it as-// projects/default/instances/default/databases/d.-func TestExplainDMLWithoutRunningIt(t *testing.T) {+// testDatabase returns the database that SPANNER_TEST_DATABASE names, as+// projects/default/instances/default/databases/d, in the Spanner that the client+// reaches without TLS or credentials through SPANNER_EMULATOR_HOST. CI provides+// no Spanner, so the test is skipped without them.+func testDatabase(t *testing.T) string {+ t.Helper()database := os.Getenv("SPANNER_TEST_DATABASE")if os.Getenv("SPANNER_EMULATOR_HOST") == "" || database == "" {- t.Skip("set SPANNER_EMULATOR_HOST and SPANNER_TEST_DATABASE to run against Spanner Omni")+ t.Skip("set SPANNER_EMULATOR_HOST and SPANNER_TEST_DATABASE to run against Spanner")}+ return database+}++// TestExplainDMLWithoutRunningIt needs a Spanner that plans queries, which the+// emulator does not do and Spanner Omni does.+func TestExplainDMLWithoutRunningIt(t *testing.T) {+ database := testDatabase(t)a := require.New(t)ctx := context.Background()@@ -74,6 +82,108 @@ func TestExplainDMLWithoutRunningIt(t *testing.T) {a.Equal(int64(10), total)}+func TestQueryConnLimitsQueryStartingWithWith(t *testing.T) {+ a := require.New(t)+ ctx := context.Background()+ client, err := spanner.NewClient(ctx, testDatabase(t))+ a.NoError(err)+ defer client.Close()++ driver := &Driver{client: client}+ for _, statement := range []string{+ "WITH n AS (SELECT x FROM UNNEST([3, 1, 2]) AS x) SELECT x FROM n ORDER BY x",+ // The rewrite keeps a non-literal LIMIT, so the driver caps the rows.+ "WITH n AS (SELECT x FROM UNNEST([3, 1, 2]) AS x) SELECT x FROM n ORDER BY x LIMIT CAST(3 AS INT64)",+ } {+ results, err := driver.QueryConn(ctx, nil, statement, db.QueryContext{+ Limit: 2,+ MaximumSQLResultSize: math.MaxInt64,+ })+ a.NoError(err)+ a.Len(results, 1)+ a.Empty(results[0].Error, results[0].Statement)+ var values []int64+ for _, row := range results[0].Rows {+ values = append(values, row.Values[0].GetInt64Value())+ }+ a.Equal([]int64{1, 2}, values, results[0].Statement)+ }+}++func TestGetStatementWithResultLimit(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want string+ }{+ {+ name: "select",+ statement: "SELECT * FROM t",+ want: "SELECT * FROM t LIMIT 100",+ },+ {+ name: "with",+ statement: "WITH a AS (SELECT 1 AS x) SELECT x FROM a",+ want: "WITH a AS (SELECT 1 AS x) SELECT x FROM a LIMIT 100",+ },+ {+ name: "lowercase with, order by, and trailing semicolon",+ statement: "with a as (select 1 x), b as (select x from a) select x from b order by x desc;",+ want: "with a as (select 1 x), b as (select x from a) select x from b order by x desc LIMIT 100;",+ },+ {+ name: "with and set operation",+ statement: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x",+ want: "WITH a AS (SELECT 1 AS x) (SELECT x FROM a) UNION ALL (SELECT x FROM a) ORDER BY x LIMIT 100",+ },+ {+ name: "set operation",+ statement: "SELECT 1 UNION ALL SELECT 2",+ want: "SELECT 1 UNION ALL SELECT 2 LIMIT 100",+ },+ {+ name: "inner limit belongs to the parenthesized query",+ statement: "(SELECT x FROM t LIMIT 500)",+ want: "(SELECT x FROM t LIMIT 500) LIMIT 100",+ },+ {+ name: "trailing comment",+ statement: "SELECT x FROM t -- note",+ want: "SELECT x FROM t LIMIT 100 -- note",+ },+ {+ name: "lower limit is kept",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT 10",+ },+ {+ name: "higher limit is lowered",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 5000 OFFSET 10",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a ORDER BY x LIMIT 100 OFFSET 10",+ },+ {+ name: "non-literal limit is unchanged",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a LIMIT CAST(5000 AS INT64)",+ },+ {+ name: "for update is unchanged",+ statement: "SELECT x FROM t ORDER BY x FOR UPDATE",+ want: "SELECT x FROM t ORDER BY x FOR UPDATE",+ },+ {+ name: "unparsable query is unchanged",+ statement: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",+ want: "WITH a AS (SELECT x FROM t) SELECT x FROM a |> WHERE x > 1",+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ require.Equal(t, tc.want, getStatementWithResultLimit(tc.statement, 100))+ })+ }+}+func TestGetDatabaseFromDSN(t *testing.T) {tests := []struct {dsn string