Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

test(cosmosdb): cover cross-partition queries against a local emulator (#21885)

Danny Xu2f42e0947994
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.go
index e98bc98854..5ec45a9bef 100644
--- a/backend/common/testcontainer/shared.go
+++ b/backend/common/testcontainer/shared.go
@@ -31,6 +31,7 @@ var (
// 10.7 is the oldest MariaDB release Bytebase supports.
sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))
sharedRedisCluster = startOnce(getRedisClusterContainer)
+ sharedCosmos = startOnce(getCosmosContainer)
startedMu sync.Mutex
started []*Container
@@ -136,6 +137,14 @@ func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, share
// slot, for the driver paths that need a *redis.ClusterClient.
func SharedRedisClusterContainer(t testing.TB) *Container { return shared(t, sharedRedisCluster) }
+// SharedCosmosContainer is SharedPgContainer for the Cosmos DB emulator. It
+// carries no *sql.DB, so a test builds its endpoint from GetHost and GetPort
+// and authenticates with the well-known emulator key. Tests share its
+// databases rather than taking one each: the emulator hands out a fixed budget
+// of physical partitions, and a container wide enough to span several of them
+// spends a large part of that budget.
+func SharedCosmosContainer(t testing.TB) *Container { return shared(t, sharedCosmos) }
+
// NewPgDatabase creates an empty database for the test on the shared
// PostgreSQL container and returns its name with a superuser handle to it.
// Anything cluster-wide the test creates, such as a role, should be named
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.go
index 4c9e0c0345..c03ac45817 100644
--- a/backend/common/testcontainer/testcontainer.go
+++ b/backend/common/testcontainer/testcontainer.go
@@ -5,6 +5,7 @@ import (
"context"
"crypto/rand"
"crypto/rsa"
+ "crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"database/sql"
@@ -20,6 +21,8 @@ import (
"strings"
"time"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/network"
"github.com/pkg/errors"
goora "github.com/sijms/go-ora/v2"
"github.com/testcontainers/testcontainers-go"
@@ -478,6 +481,58 @@ func createTLSMaterial(ips ...net.IP) (string, string, []byte, []byte, error) {
return tlsDir, caPath, certificate, key, nil
}
+// CosmosEmulatorPort is the port the Cosmos emulator serves and advertises.
+// It is published unchanged, so only one emulator can run on a machine.
+const CosmosEmulatorPort = "8081"
+
+// getCosmosContainer starts the classic Cosmos DB emulator. It is the only
+// image that reproduces the gateway refusing to serve a cross-partition query,
+// which is the error the SDK pivots to its query engine on: the vnext-preview
+// emulator answers those queries itself out of PostgreSQL and reports one
+// partition key range whatever the throughput, so a query engine never runs.
+//
+// The emulator names its own endpoint in the database account document and the
+// SDK follows that rather than the endpoint it was configured with. The address
+// override makes it advertise 127.0.0.1, which only reaches the emulator while
+// the host publishes the same port it serves — hence the fixed binding, and one
+// emulator per machine. A second concurrent start fails on that binding instead
+// of sharing the first one's databases.
+//
+// Partition count is the emulator's budget of physical partitions to hand out;
+// a container draws from it in proportion to its provisioned throughput, so 8
+// leaves room for a fixture wide enough to exercise a cross-partition merge.
+func getCosmosContainer(ctx context.Context) (*Container, error) {
+ emulatorPort, err := network.ParsePort(CosmosEmulatorPort + "/tcp")
+ if err != nil {
+ return nil, errors.Wrap(err, "parse the Cosmos emulator port")
+ }
+ return start(ctx, testcontainers.ContainerRequest{
+ Image: "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:2.14.28",
+ Env: map[string]string{
+ "AZURE_COSMOS_EMULATOR_PARTITION_COUNT": "8",
+ "AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE": "false",
+ "AZURE_COSMOS_EMULATOR_IP_ADDRESS_OVERRIDE": "127.0.0.1",
+ },
+ // Readiness is an answer from the gateway rather than a log line: the
+ // emulator logs "Started N/M partitions" per partition before the bare
+ // "Started" that means it is up, and terminates its lines with CRLF.
+ //
+ // Bringing up the partitions takes about half a minute on an idle host and
+ // was measured at 216 s on a loaded one, so the timeout clears the longest
+ // boot seen rather than the typical one.
+ WaitingFor: wait.ForHTTP("/_explorer/emulator.pem").
+ WithPort(CosmosEmulatorPort+"/tcp").
+ WithTLS(true, &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ InsecureSkipVerify: true, //nolint:gosec // the emulator signs its certificate itself
+ }).
+ WithStartupTimeout(10 * time.Minute),
+ HostConfigModifier: func(hc *container.HostConfig) {
+ hc.PortBindings = network.PortMap{emulatorPort: {{HostPort: CosmosEmulatorPort}}}
+ },
+ }, spec{port: CosmosEmulatorPort + "/tcp"})
+}
+
// RedisClusterSlots is the number of hash slots a Redis cluster partitions keys
// into; the single node getRedisClusterContainer starts owns all of them.
const RedisClusterSlots = 16384
diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.go
index 6124438137..d9edc8f189 100644
--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go
+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go
@@ -1,14 +1,24 @@
// Copyright (c) Bytebase (Hong Kong) Limited.
package cosmosdb
-// End-to-end integration tests for BYT-9239 — every query class unlocked by
-// Stages 1 through 6 of the pure-Go Cosmos query engine, exercised through
-// the Bytebase driver's QueryConn path against a live Cosmos account.
+// Two suites cover the cross-partition query path, the one thing the forked
+// azcosmos SDK adds over upstream: the gateway cannot serve an aggregate,
+// DISTINCT, ORDER BY, TOP, OFFSET/LIMIT or GROUP BY that spans partitions, so
+// NewCrossPartitionQueryItemsPager pivots to a query engine that merges the
+// per-partition streams itself.
//
-// Guarded by AZURE_COSMOS_KEY — skips automatically when unset, so default
-// unit-test runs stay hermetic.
+// TestCrossPartition* run on the emulator SharedCosmosContainer starts, against
+// a fixture whose contents are known, and so assert exact answers. Both suites
+// are manual: this one starts a 4.9 GB emulator image that takes most of a
+// minute to boot and publishes a fixed port, so only one can run on a machine.
//
-// Running locally against the bytebase-cosmostest account:
+// AZURE_COSMOS_EMULATOR=1 go test -count=1 -run "^TestCrossPartition" \
+// -timeout 600s ./backend/plugin/db/cosmosdb/
+//
+// TestIntegration_BYT9239_* run the same query classes against a live Cosmos
+// account, which the emulator cannot stand in for entirely, and assert only
+// what holds for data they do not control. They are guarded by
+// AZURE_COSMOS_KEY and skip when it is unset:
//
// export AZURE_COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"
// export AZURE_COSMOS_KEY="$(az cosmosdb keys list \
@@ -21,17 +31,29 @@ package cosmosdb
import (
"context"
"encoding/json"
+ "fmt"
+ "net/http"
"os"
+ "sync"
"testing"
"time"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"
+ "github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
+
+ // Registers the CosmosDB query validator that QueryConn calls.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/cosmosdb"
)
+func TestMain(m *testing.M) { testcontainer.Main(m) }
+
func skipIfNoAzure(t *testing.T) (endpoint, key, dbName, container string) {
t.Helper()
endpoint = os.Getenv("AZURE_COSMOS_ENDPOINT")
@@ -397,3 +419,302 @@ func cosmosTypeRank(v any) (int, any) {
return 0, x
}
}
+
+// ------------------------------------------- cross-partition, local emulator
+
+const (
+ crossPartitionDatabase = "crosspartition"
+ crossPartitionContainer = "cities"
+ // The emulator hands a container one physical partition per 10k RU, so
+ // throughput — not document count — is what makes these queries span
+ // partitions and reach the merge the query engine exists for.
+ crossPartitionThroughput = 30000
+ crossPartitionCitiesPerCountry = 5
+ crossPartitionPopulationGap = 100000
+)
+
+// crossPartitionCountries are the fixture's partition key values. Each holds
+// crossPartitionCitiesPerCountry cities whose populations step by
+// crossPartitionPopulationGap, so every aggregate has one correct answer.
+var crossPartitionCountries = []string{"AE", "BR", "CN", "DE", "FR", "IN", "JP", "US"}
+
+func crossPartitionItems() int { return len(crossPartitionCountries) * crossPartitionCitiesPerCountry }
+
+// crossPartitionPopulationSum is the total the fixture's SUM must return.
+func crossPartitionPopulationSum() int {
+ perCountry := 0
+ for i := 1; i <= crossPartitionCitiesPerCountry; i++ {
+ perCountry += i * crossPartitionPopulationGap
+ }
+ return perCountry * len(crossPartitionCountries)
+}
+
+type crossPartitionCity struct {
+ ID string `json:"id"`
+ Country string `json:"country"`
+ Name string `json:"name"`
+ Population int `json:"population"`
+}
+
+var (
+ crossPartitionOnce sync.Once
+ crossPartitionDriver *Driver
+ crossPartitionRanges int
+ crossPartitionErr error
+)
+
+// crossPartitionEnv opts into the cross-partition suite. Gating it here rather
+// than in each test is what keeps the emulator from starting at all on a run
+// that did not ask for it, since SharedCosmosContainer starts lazily.
+const crossPartitionEnv = "AZURE_COSMOS_EMULATOR"
+
+// emulatorFixture returns a driver pointed at the seeded fixture, building it
+// on first use. It refuses a fixture confined to one physical partition: the
+// gateway serves those queries on its own, so the tests below would pass
+// without the query engine ever running.
+func emulatorFixture(t *testing.T) *Driver {
+ t.Helper()
+ if os.Getenv(crossPartitionEnv) != "1" {
+ t.Skipf("set %s=1 to run the cross-partition tests against a local Cosmos emulator", crossPartitionEnv)
+ }
+ container := testcontainer.SharedCosmosContainer(t)
+ endpoint := fmt.Sprintf("https://%s:%s", container.GetHost(), container.GetPort())
+ crossPartitionOnce.Do(func() {
+ crossPartitionDriver, crossPartitionRanges, crossPartitionErr = seedCrossPartition(endpoint)
+ })
+ require.NoError(t, crossPartitionErr)
+ require.Greater(t, crossPartitionRanges, 1,
+ "fixture spans %d physical partition(s); cross-partition queries need more than one", crossPartitionRanges)
+ return crossPartitionDriver
+}
+
+func seedCrossPartition(endpoint string) (*Driver, int, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+
+ opened, err := (&Driver{}).Open(ctx, storepb.Engine_COSMOSDB, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{Host: endpoint},
+ ConnectionContext: db.ConnectionContext{DatabaseName: crossPartitionDatabase},
+ })
+ if err != nil {
+ return nil, 0, errors.Wrap(err, "open the driver against the emulator")
+ }
+ driver, ok := opened.(*Driver)
+ if !ok {
+ return nil, 0, errors.Errorf("Open returned %T, want *Driver", opened)
+ }
+ if err := driver.Ping(ctx); err != nil {
+ return nil, 0, errors.Wrap(err, "ping the emulator")
+ }
+
+ if _, err := driver.client.CreateDatabase(ctx, azcosmos.DatabaseProperties{ID: crossPartitionDatabase}, nil); err != nil && !alreadyExists(err) {
+ return nil, 0, errors.Wrap(err, "create the fixture database")
+ }
+ database, err := driver.client.NewDatabase(crossPartitionDatabase)
+ if err != nil {
+ return nil, 0, errors.Wrap(err, "open the fixture database")
+ }
+ throughput := azcosmos.NewManualThroughputProperties(crossPartitionThroughput)
+ if _, err := database.CreateContainer(ctx, azcosmos.ContainerProperties{
+ ID: crossPartitionContainer,
+ PartitionKeyDefinition: azcosmos.PartitionKeyDefinition{Paths: []string{"/country"}},
+ }, &azcosmos.CreateContainerOptions{ThroughputProperties: &throughput}); err != nil && !alreadyExists(err) {
+ return nil, 0, errors.Wrap(err, "create the fixture container")
+ }
+ container, err := database.NewContainer(crossPartitionContainer)
+ if err != nil {
+ return nil, 0, errors.Wrap(err, "open the fixture container")
+ }
+
+ for _, country := range crossPartitionCountries {
+ for i := range crossPartitionCitiesPerCountry {
+ item, err := json.Marshal(crossPartitionCity{
+ ID: fmt.Sprintf("%s-%d", country, i),
+ Country: country,
+ Name: fmt.Sprintf("city-%s-%d", country, i),
+ Population: (i + 1) * crossPartitionPopulationGap,
+ })
+ if err != nil {
+ return nil, 0, errors.Wrap(err, "marshal a fixture city")
+ }
+ if _, err := container.CreateItem(ctx, azcosmos.NewPartitionKeyString(country), item, nil); err != nil && !alreadyExists(err) {
+ return nil, 0, errors.Wrapf(err, "seed %s-%d", country, i)
+ }
+ }
+ }
+
+ ranges, err := countPartitionKeyRanges(endpoint)
+ if err != nil {
+ return nil, 0, err
+ }
+ return driver, ranges, nil
+}
+
+// countPartitionKeyRanges reports how many physical partitions the fixture
+// container spans. It reads the REST endpoint because the SDK keeps the range
+// list to itself, inside the query path under test.
+func countPartitionKeyRanges(endpoint string) (int, error) {
+ client, err := newEmulatorRESTClient(endpoint)
+ if err != nil {
+ return 0, errors.Wrap(err, "build the emulator REST client")
+ }
+ link := fmt.Sprintf("dbs/%s/colls/%s", crossPartitionDatabase, crossPartitionContainer)
+ body, err := client.get(link+"/pkranges", "pkranges", link)
+ if err != nil {
+ return 0, errors.Wrap(err, "read the container's partition key ranges")
+ }
+ var resp struct {
+ PartitionKeyRanges []struct {
+ ID string `json:"id"`
+ } `json:"PartitionKeyRanges"`
+ }
+ if err := json.Unmarshal(body, &resp); err != nil {
+ return 0, errors.Wrap(err, "parse the partition key range list")
+ }
+ return len(resp.PartitionKeyRanges), nil
+}
+
+func alreadyExists(err error) bool {
+ var responseErr *azcore.ResponseError
+ return errors.As(err, &responseErr) && responseErr.StatusCode == http.StatusConflict
+}
+
+func TestCrossPartitionAggregates(t *testing.T) {
+ driver := emulatorFixture(t)
+ items := crossPartitionItems()
+ sum := crossPartitionPopulationSum()
+
+ for _, tc := range []struct {
+ name string
+ sql string
+ want string
+ }{
+ {"count_value", `SELECT VALUE COUNT(1) FROM c`, fmt.Sprint(items)},
+ {"count_aliased", `SELECT COUNT(1) AS total FROM c`, fmt.Sprintf(`{"total":%d}`, items)},
+ {"sum_value", `SELECT VALUE SUM(c.population) FROM c`, fmt.Sprint(sum)},
+ {"avg_value", `SELECT VALUE AVG(c.population) FROM c`, fmt.Sprint(sum / items)},
+ {"min_max_aliased", `SELECT MIN(c.population) AS lo, MAX(c.population) AS hi FROM c`,
+ fmt.Sprintf(`{"lo":%d,"hi":%d}`, crossPartitionPopulationGap, crossPartitionCitiesPerCountry*crossPartitionPopulationGap)},
+ {"count_and_sum_aliased", `SELECT COUNT(1) AS total, SUM(c.population) AS pop FROM c`,
+ fmt.Sprintf(`{"total":%d,"pop":%d}`, items, sum)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, tc.sql)
+ require.Len(t, rows, 1)
+ assert.JSONEq(t, tc.want, string(rows[0]))
+ })
+ }
+}
+
+func TestCrossPartitionDistinct(t *testing.T) {
+ driver := emulatorFixture(t)
+
+ t.Run("value_form", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT VALUE c.country FROM c`)
+ got := make([]string, 0, len(rows))
+ for _, row := range rows {
+ var country string
+ require.NoError(t, json.Unmarshal(row, &country))
+ got = append(got, country)
+ }
+ assert.ElementsMatch(t, crossPartitionCountries, got)
+ })
+
+ t.Run("object_form", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT c.country FROM c`)
+ assert.Len(t, rows, len(crossPartitionCountries))
+ })
+}
+
+func TestCrossPartitionOrderBy(t *testing.T) {
+ driver := emulatorFixture(t)
+
+ t.Run("ascending", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT c.name, c.population FROM c ORDER BY c.population ASC`)
+ require.Len(t, rows, crossPartitionItems())
+ assertOrderedByPopulation(t, rows, true)
+ })
+
+ t.Run("descending", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT c.name, c.population FROM c ORDER BY c.population DESC`)
+ require.Len(t, rows, crossPartitionItems())
+ assertOrderedByPopulation(t, rows, false)
+ })
+}
+
+func TestCrossPartitionTop(t *testing.T) {
+ driver := emulatorFixture(t)
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT TOP 3 * FROM c`)
+ assert.Len(t, rows, 3)
+}
+
+// TestCrossPartitionOffsetLimit pins the page against the globally ordered
+// result: a merge that drops or reorders a partition's stream still returns
+// five rows, so only comparing the slice catches it.
+func TestCrossPartitionOffsetLimit(t *testing.T) {
+ driver := emulatorFixture(t)
+
+ all := assertAscendingByName(t, runQuery(t, driver, crossPartitionContainer, `SELECT c.name FROM c ORDER BY c.name`))
+ require.Len(t, all, crossPartitionItems())
+
+ page := assertAscendingByName(t, runQuery(t, driver, crossPartitionContainer, `SELECT c.name FROM c ORDER BY c.name OFFSET 5 LIMIT 5`))
+ assert.Equal(t, all[5:10], page)
+}
+
+func TestCrossPartitionGroupBy(t *testing.T) {
+ driver := emulatorFixture(t)
+ rows := runQuery(t, driver, crossPartitionContainer,
+ `SELECT c.country, COUNT(1) AS cities, SUM(c.population) AS pop FROM c GROUP BY c.country`)
+ require.Len(t, rows, len(crossPartitionCountries))
+
+ wantPop := float64(crossPartitionPopulationSum() / len(crossPartitionCountries))
+ seen := map[string]bool{}
+ for _, row := range rows {
+ var group struct {
+ Country string `json:"country"`
+ Cities float64 `json:"cities"`
+ Pop float64 `json:"pop"`
+ }
+ require.NoError(t, json.Unmarshal(row, &group))
+ assert.False(t, seen[group.Country], "country %q grouped twice", group.Country)
+ seen[group.Country] = true
+ assert.Equal(t, float64(crossPartitionCitiesPerCountry), group.Cities, "country %q", group.Country)
+ assert.Equal(t, wantPop, group.Pop, "country %q", group.Country)
+ }
+ assert.Len(t, seen, len(crossPartitionCountries))
+}
+
+// TestCrossPartitionGatewayServable covers the queries that never leave the
+// gateway path, which the engine must not take over: a projection of every
+// partition, and a filter the gateway can route to a single one.
+func TestCrossPartitionGatewayServable(t *testing.T) {
+ driver := emulatorFixture(t)
+
+ t.Run("projection", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT * FROM c`)
+ assert.Len(t, rows, crossPartitionItems())
+ })
+
+ t.Run("single_partition_filter", func(t *testing.T) {
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT * FROM c WHERE c.country = "US"`)
+ assert.Len(t, rows, crossPartitionCitiesPerCountry)
+ })
+}
+
+// TestCrossPartitionQueryLimit covers QueryConn's own row cap, which stops
+// paging mid-result and so decides how much of a cross-partition query the
+// driver pulls back.
+func TestCrossPartitionQueryLimit(t *testing.T) {
+ driver := emulatorFixture(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ results, err := driver.QueryConn(ctx, nil, `SELECT * FROM c`, db.QueryContext{
+ Container: crossPartitionContainer,
+ Limit: 7,
+ })
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ assert.Len(t, results[0].Rows, 7)
+ assert.EqualValues(t, 7, results[0].RowsCount)
+}
diff --git a/backend/plugin/db/util/pgsession.go b/backend/plugin/db/util/pgsession.go
index a3de194865..c66595c606 100644
--- a/backend/plugin/db/util/pgsession.go
+++ b/backend/plugin/db/util/pgsession.go
@@ -1,3 +1,4 @@
+//nolint:revive
package util
import (
diff --git a/backend/plugin/db/util/pgsession_test.go b/backend/plugin/db/util/pgsession_test.go
index d1d79ca9b3..fca7158f8f 100644
--- a/backend/plugin/db/util/pgsession_test.go
+++ b/backend/plugin/db/util/pgsession_test.go
@@ -1,3 +1,4 @@
+//nolint:revive
package util
import (
diff --git a/go.mod b/go.mod
index 107e5f4991..d42bf76b23 100644
--- a/go.mod
+++ b/go.mod
@@ -70,6 +70,7 @@ require (
github.com/lib/pq v1.12.3
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8
github.com/microsoft/go-mssqldb v1.11.2
+ github.com/moby/moby/api v1.56.1
github.com/modelcontextprotocol/go-sdk v1.8.0
github.com/nyaruka/phonenumbers/v2 v2.0.16
github.com/openark/golib v0.0.0-20210531070646-355f37940af8
@@ -270,7 +271,6 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.3.0 // indirect
- github.com/moby/moby/api v1.56.1 // indirect
github.com/moby/moby/client v0.6.2 // indirect
github.com/moby/patternmatcher v0.6.1 // indirect
github.com/moby/sys/sequential v0.7.0 // indirect