Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
test(cosmosdb): cover cross-partition queries against a local emulator (#21885)
Danny Xu2f42e0947994
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.goindex e98bc98854..5ec45a9bef 100644--- a/backend/common/testcontainer/shared.go+++ b/backend/common/testcontainer/shared.go@@ -31,6 +31,7 @@ var (// 10.7 is the oldest MariaDB release Bytebase supports.sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))sharedRedisCluster = startOnce(getRedisClusterContainer)+ sharedCosmos = startOnce(getCosmosContainer)startedMu sync.Mutexstarted []*Container@@ -136,6 +137,14 @@ func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, share// slot, for the driver paths that need a *redis.ClusterClient.func SharedRedisClusterContainer(t testing.TB) *Container { return shared(t, sharedRedisCluster) }+// SharedCosmosContainer is SharedPgContainer for the Cosmos DB emulator. It+// carries no *sql.DB, so a test builds its endpoint from GetHost and GetPort+// and authenticates with the well-known emulator key. Tests share its+// databases rather than taking one each: the emulator hands out a fixed budget+// of physical partitions, and a container wide enough to span several of them+// spends a large part of that budget.+func SharedCosmosContainer(t testing.TB) *Container { return shared(t, sharedCosmos) }+// NewPgDatabase creates an empty database for the test on the shared// PostgreSQL container and returns its name with a superuser handle to it.// Anything cluster-wide the test creates, such as a role, should be nameddiff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 4c9e0c0345..c03ac45817 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -5,6 +5,7 @@ import ("context""crypto/rand""crypto/rsa"+ "crypto/tls""crypto/x509""crypto/x509/pkix""database/sql"@@ -20,6 +21,8 @@ import ("strings""time"+ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/network""github.com/pkg/errors"goora "github.com/sijms/go-ora/v2""github.com/testcontainers/testcontainers-go"@@ -478,6 +481,58 @@ func createTLSMaterial(ips ...net.IP) (string, string, []byte, []byte, error) {return tlsDir, caPath, certificate, key, nil}+// CosmosEmulatorPort is the port the Cosmos emulator serves and advertises.+// It is published unchanged, so only one emulator can run on a machine.+const CosmosEmulatorPort = "8081"++// getCosmosContainer starts the classic Cosmos DB emulator. It is the only+// image that reproduces the gateway refusing to serve a cross-partition query,+// which is the error the SDK pivots to its query engine on: the vnext-preview+// emulator answers those queries itself out of PostgreSQL and reports one+// partition key range whatever the throughput, so a query engine never runs.+//+// The emulator names its own endpoint in the database account document and the+// SDK follows that rather than the endpoint it was configured with. The address+// override makes it advertise 127.0.0.1, which only reaches the emulator while+// the host publishes the same port it serves — hence the fixed binding, and one+// emulator per machine. A second concurrent start fails on that binding instead+// of sharing the first one's databases.+//+// Partition count is the emulator's budget of physical partitions to hand out;+// a container draws from it in proportion to its provisioned throughput, so 8+// leaves room for a fixture wide enough to exercise a cross-partition merge.+func getCosmosContainer(ctx context.Context) (*Container, error) {+ emulatorPort, err := network.ParsePort(CosmosEmulatorPort + "/tcp")+ if err != nil {+ return nil, errors.Wrap(err, "parse the Cosmos emulator port")+ }+ return start(ctx, testcontainers.ContainerRequest{+ Image: "mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:2.14.28",+ Env: map[string]string{+ "AZURE_COSMOS_EMULATOR_PARTITION_COUNT": "8",+ "AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE": "false",+ "AZURE_COSMOS_EMULATOR_IP_ADDRESS_OVERRIDE": "127.0.0.1",+ },+ // Readiness is an answer from the gateway rather than a log line: the+ // emulator logs "Started N/M partitions" per partition before the bare+ // "Started" that means it is up, and terminates its lines with CRLF.+ //+ // Bringing up the partitions takes about half a minute on an idle host and+ // was measured at 216 s on a loaded one, so the timeout clears the longest+ // boot seen rather than the typical one.+ WaitingFor: wait.ForHTTP("/_explorer/emulator.pem").+ WithPort(CosmosEmulatorPort+"/tcp").+ WithTLS(true, &tls.Config{+ MinVersion: tls.VersionTLS12,+ InsecureSkipVerify: true, //nolint:gosec // the emulator signs its certificate itself+ }).+ WithStartupTimeout(10 * time.Minute),+ HostConfigModifier: func(hc *container.HostConfig) {+ hc.PortBindings = network.PortMap{emulatorPort: {{HostPort: CosmosEmulatorPort}}}+ },+ }, spec{port: CosmosEmulatorPort + "/tcp"})+}+// RedisClusterSlots is the number of hash slots a Redis cluster partitions keys// into; the single node getRedisClusterContainer starts owns all of them.const RedisClusterSlots = 16384diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.goindex 6124438137..d9edc8f189 100644--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go@@ -1,14 +1,24 @@// Copyright (c) Bytebase (Hong Kong) Limited.package cosmosdb-// End-to-end integration tests for BYT-9239 — every query class unlocked by-// Stages 1 through 6 of the pure-Go Cosmos query engine, exercised through-// the Bytebase driver's QueryConn path against a live Cosmos account.+// Two suites cover the cross-partition query path, the one thing the forked+// azcosmos SDK adds over upstream: the gateway cannot serve an aggregate,+// DISTINCT, ORDER BY, TOP, OFFSET/LIMIT or GROUP BY that spans partitions, so+// NewCrossPartitionQueryItemsPager pivots to a query engine that merges the+// per-partition streams itself.//-// Guarded by AZURE_COSMOS_KEY — skips automatically when unset, so default-// unit-test runs stay hermetic.+// TestCrossPartition* run on the emulator SharedCosmosContainer starts, against+// a fixture whose contents are known, and so assert exact answers. Both suites+// are manual: this one starts a 4.9 GB emulator image that takes most of a+// minute to boot and publishes a fixed port, so only one can run on a machine.//-// Running locally against the bytebase-cosmostest account:+// AZURE_COSMOS_EMULATOR=1 go test -count=1 -run "^TestCrossPartition" \+// -timeout 600s ./backend/plugin/db/cosmosdb/+//+// TestIntegration_BYT9239_* run the same query classes against a live Cosmos+// account, which the emulator cannot stand in for entirely, and assert only+// what holds for data they do not control. They are guarded by+// AZURE_COSMOS_KEY and skip when it is unset://// export AZURE_COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"// export AZURE_COSMOS_KEY="$(az cosmosdb keys list \@@ -21,17 +31,29 @@ package cosmosdbimport ("context""encoding/json"+ "fmt"+ "net/http""os"+ "sync""testing""time"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"+ "github.com/pkg/errors""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db"++ // Registers the CosmosDB query validator that QueryConn calls.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/cosmosdb")+func TestMain(m *testing.M) { testcontainer.Main(m) }+func skipIfNoAzure(t *testing.T) (endpoint, key, dbName, container string) {t.Helper()endpoint = os.Getenv("AZURE_COSMOS_ENDPOINT")@@ -397,3 +419,302 @@ func cosmosTypeRank(v any) (int, any) {return 0, x}}++// ------------------------------------------- cross-partition, local emulator++const (+ crossPartitionDatabase = "crosspartition"+ crossPartitionContainer = "cities"+ // The emulator hands a container one physical partition per 10k RU, so+ // throughput — not document count — is what makes these queries span+ // partitions and reach the merge the query engine exists for.+ crossPartitionThroughput = 30000+ crossPartitionCitiesPerCountry = 5+ crossPartitionPopulationGap = 100000+)++// crossPartitionCountries are the fixture's partition key values. Each holds+// crossPartitionCitiesPerCountry cities whose populations step by+// crossPartitionPopulationGap, so every aggregate has one correct answer.+var crossPartitionCountries = []string{"AE", "BR", "CN", "DE", "FR", "IN", "JP", "US"}++func crossPartitionItems() int { return len(crossPartitionCountries) * crossPartitionCitiesPerCountry }++// crossPartitionPopulationSum is the total the fixture's SUM must return.+func crossPartitionPopulationSum() int {+ perCountry := 0+ for i := 1; i <= crossPartitionCitiesPerCountry; i++ {+ perCountry += i * crossPartitionPopulationGap+ }+ return perCountry * len(crossPartitionCountries)+}++type crossPartitionCity struct {+ ID string `json:"id"`+ Country string `json:"country"`+ Name string `json:"name"`+ Population int `json:"population"`+}++var (+ crossPartitionOnce sync.Once+ crossPartitionDriver *Driver+ crossPartitionRanges int+ crossPartitionErr error+)++// crossPartitionEnv opts into the cross-partition suite. Gating it here rather+// than in each test is what keeps the emulator from starting at all on a run+// that did not ask for it, since SharedCosmosContainer starts lazily.+const crossPartitionEnv = "AZURE_COSMOS_EMULATOR"++// emulatorFixture returns a driver pointed at the seeded fixture, building it+// on first use. It refuses a fixture confined to one physical partition: the+// gateway serves those queries on its own, so the tests below would pass+// without the query engine ever running.+func emulatorFixture(t *testing.T) *Driver {+ t.Helper()+ if os.Getenv(crossPartitionEnv) != "1" {+ t.Skipf("set %s=1 to run the cross-partition tests against a local Cosmos emulator", crossPartitionEnv)+ }+ container := testcontainer.SharedCosmosContainer(t)+ endpoint := fmt.Sprintf("https://%s:%s", container.GetHost(), container.GetPort())+ crossPartitionOnce.Do(func() {+ crossPartitionDriver, crossPartitionRanges, crossPartitionErr = seedCrossPartition(endpoint)+ })+ require.NoError(t, crossPartitionErr)+ require.Greater(t, crossPartitionRanges, 1,+ "fixture spans %d physical partition(s); cross-partition queries need more than one", crossPartitionRanges)+ return crossPartitionDriver+}++func seedCrossPartition(endpoint string) (*Driver, int, error) {+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)+ defer cancel()++ opened, err := (&Driver{}).Open(ctx, storepb.Engine_COSMOSDB, db.ConnectionConfig{+ DataSource: &storepb.DataSource{Host: endpoint},+ ConnectionContext: db.ConnectionContext{DatabaseName: crossPartitionDatabase},+ })+ if err != nil {+ return nil, 0, errors.Wrap(err, "open the driver against the emulator")+ }+ driver, ok := opened.(*Driver)+ if !ok {+ return nil, 0, errors.Errorf("Open returned %T, want *Driver", opened)+ }+ if err := driver.Ping(ctx); err != nil {+ return nil, 0, errors.Wrap(err, "ping the emulator")+ }++ if _, err := driver.client.CreateDatabase(ctx, azcosmos.DatabaseProperties{ID: crossPartitionDatabase}, nil); err != nil && !alreadyExists(err) {+ return nil, 0, errors.Wrap(err, "create the fixture database")+ }+ database, err := driver.client.NewDatabase(crossPartitionDatabase)+ if err != nil {+ return nil, 0, errors.Wrap(err, "open the fixture database")+ }+ throughput := azcosmos.NewManualThroughputProperties(crossPartitionThroughput)+ if _, err := database.CreateContainer(ctx, azcosmos.ContainerProperties{+ ID: crossPartitionContainer,+ PartitionKeyDefinition: azcosmos.PartitionKeyDefinition{Paths: []string{"/country"}},+ }, &azcosmos.CreateContainerOptions{ThroughputProperties: &throughput}); err != nil && !alreadyExists(err) {+ return nil, 0, errors.Wrap(err, "create the fixture container")+ }+ container, err := database.NewContainer(crossPartitionContainer)+ if err != nil {+ return nil, 0, errors.Wrap(err, "open the fixture container")+ }++ for _, country := range crossPartitionCountries {+ for i := range crossPartitionCitiesPerCountry {+ item, err := json.Marshal(crossPartitionCity{+ ID: fmt.Sprintf("%s-%d", country, i),+ Country: country,+ Name: fmt.Sprintf("city-%s-%d", country, i),+ Population: (i + 1) * crossPartitionPopulationGap,+ })+ if err != nil {+ return nil, 0, errors.Wrap(err, "marshal a fixture city")+ }+ if _, err := container.CreateItem(ctx, azcosmos.NewPartitionKeyString(country), item, nil); err != nil && !alreadyExists(err) {+ return nil, 0, errors.Wrapf(err, "seed %s-%d", country, i)+ }+ }+ }++ ranges, err := countPartitionKeyRanges(endpoint)+ if err != nil {+ return nil, 0, err+ }+ return driver, ranges, nil+}++// countPartitionKeyRanges reports how many physical partitions the fixture+// container spans. It reads the REST endpoint because the SDK keeps the range+// list to itself, inside the query path under test.+func countPartitionKeyRanges(endpoint string) (int, error) {+ client, err := newEmulatorRESTClient(endpoint)+ if err != nil {+ return 0, errors.Wrap(err, "build the emulator REST client")+ }+ link := fmt.Sprintf("dbs/%s/colls/%s", crossPartitionDatabase, crossPartitionContainer)+ body, err := client.get(link+"/pkranges", "pkranges", link)+ if err != nil {+ return 0, errors.Wrap(err, "read the container's partition key ranges")+ }+ var resp struct {+ PartitionKeyRanges []struct {+ ID string `json:"id"`+ } `json:"PartitionKeyRanges"`+ }+ if err := json.Unmarshal(body, &resp); err != nil {+ return 0, errors.Wrap(err, "parse the partition key range list")+ }+ return len(resp.PartitionKeyRanges), nil+}++func alreadyExists(err error) bool {+ var responseErr *azcore.ResponseError+ return errors.As(err, &responseErr) && responseErr.StatusCode == http.StatusConflict+}++func TestCrossPartitionAggregates(t *testing.T) {+ driver := emulatorFixture(t)+ items := crossPartitionItems()+ sum := crossPartitionPopulationSum()++ for _, tc := range []struct {+ name string+ sql string+ want string+ }{+ {"count_value", `SELECT VALUE COUNT(1) FROM c`, fmt.Sprint(items)},+ {"count_aliased", `SELECT COUNT(1) AS total FROM c`, fmt.Sprintf(`{"total":%d}`, items)},+ {"sum_value", `SELECT VALUE SUM(c.population) FROM c`, fmt.Sprint(sum)},+ {"avg_value", `SELECT VALUE AVG(c.population) FROM c`, fmt.Sprint(sum / items)},+ {"min_max_aliased", `SELECT MIN(c.population) AS lo, MAX(c.population) AS hi FROM c`,+ fmt.Sprintf(`{"lo":%d,"hi":%d}`, crossPartitionPopulationGap, crossPartitionCitiesPerCountry*crossPartitionPopulationGap)},+ {"count_and_sum_aliased", `SELECT COUNT(1) AS total, SUM(c.population) AS pop FROM c`,+ fmt.Sprintf(`{"total":%d,"pop":%d}`, items, sum)},+ } {+ t.Run(tc.name, func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, tc.sql)+ require.Len(t, rows, 1)+ assert.JSONEq(t, tc.want, string(rows[0]))+ })+ }+}++func TestCrossPartitionDistinct(t *testing.T) {+ driver := emulatorFixture(t)++ t.Run("value_form", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT VALUE c.country FROM c`)+ got := make([]string, 0, len(rows))+ for _, row := range rows {+ var country string+ require.NoError(t, json.Unmarshal(row, &country))+ got = append(got, country)+ }+ assert.ElementsMatch(t, crossPartitionCountries, got)+ })++ t.Run("object_form", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT c.country FROM c`)+ assert.Len(t, rows, len(crossPartitionCountries))+ })+}++func TestCrossPartitionOrderBy(t *testing.T) {+ driver := emulatorFixture(t)++ t.Run("ascending", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT c.name, c.population FROM c ORDER BY c.population ASC`)+ require.Len(t, rows, crossPartitionItems())+ assertOrderedByPopulation(t, rows, true)+ })++ t.Run("descending", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT c.name, c.population FROM c ORDER BY c.population DESC`)+ require.Len(t, rows, crossPartitionItems())+ assertOrderedByPopulation(t, rows, false)+ })+}++func TestCrossPartitionTop(t *testing.T) {+ driver := emulatorFixture(t)+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT TOP 3 * FROM c`)+ assert.Len(t, rows, 3)+}++// TestCrossPartitionOffsetLimit pins the page against the globally ordered+// result: a merge that drops or reorders a partition's stream still returns+// five rows, so only comparing the slice catches it.+func TestCrossPartitionOffsetLimit(t *testing.T) {+ driver := emulatorFixture(t)++ all := assertAscendingByName(t, runQuery(t, driver, crossPartitionContainer, `SELECT c.name FROM c ORDER BY c.name`))+ require.Len(t, all, crossPartitionItems())++ page := assertAscendingByName(t, runQuery(t, driver, crossPartitionContainer, `SELECT c.name FROM c ORDER BY c.name OFFSET 5 LIMIT 5`))+ assert.Equal(t, all[5:10], page)+}++func TestCrossPartitionGroupBy(t *testing.T) {+ driver := emulatorFixture(t)+ rows := runQuery(t, driver, crossPartitionContainer,+ `SELECT c.country, COUNT(1) AS cities, SUM(c.population) AS pop FROM c GROUP BY c.country`)+ require.Len(t, rows, len(crossPartitionCountries))++ wantPop := float64(crossPartitionPopulationSum() / len(crossPartitionCountries))+ seen := map[string]bool{}+ for _, row := range rows {+ var group struct {+ Country string `json:"country"`+ Cities float64 `json:"cities"`+ Pop float64 `json:"pop"`+ }+ require.NoError(t, json.Unmarshal(row, &group))+ assert.False(t, seen[group.Country], "country %q grouped twice", group.Country)+ seen[group.Country] = true+ assert.Equal(t, float64(crossPartitionCitiesPerCountry), group.Cities, "country %q", group.Country)+ assert.Equal(t, wantPop, group.Pop, "country %q", group.Country)+ }+ assert.Len(t, seen, len(crossPartitionCountries))+}++// TestCrossPartitionGatewayServable covers the queries that never leave the+// gateway path, which the engine must not take over: a projection of every+// partition, and a filter the gateway can route to a single one.+func TestCrossPartitionGatewayServable(t *testing.T) {+ driver := emulatorFixture(t)++ t.Run("projection", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT * FROM c`)+ assert.Len(t, rows, crossPartitionItems())+ })++ t.Run("single_partition_filter", func(t *testing.T) {+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT * FROM c WHERE c.country = "US"`)+ assert.Len(t, rows, crossPartitionCitiesPerCountry)+ })+}++// TestCrossPartitionQueryLimit covers QueryConn's own row cap, which stops+// paging mid-result and so decides how much of a cross-partition query the+// driver pulls back.+func TestCrossPartitionQueryLimit(t *testing.T) {+ driver := emulatorFixture(t)+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ results, err := driver.QueryConn(ctx, nil, `SELECT * FROM c`, db.QueryContext{+ Container: crossPartitionContainer,+ Limit: 7,+ })+ require.NoError(t, err)+ require.Len(t, results, 1)+ assert.Len(t, results[0].Rows, 7)+ assert.EqualValues(t, 7, results[0].RowsCount)+}diff --git a/backend/plugin/db/util/pgsession.go b/backend/plugin/db/util/pgsession.goindex a3de194865..c66595c606 100644--- a/backend/plugin/db/util/pgsession.go+++ b/backend/plugin/db/util/pgsession.go@@ -1,3 +1,4 @@+//nolint:revivepackage utilimport (diff --git a/backend/plugin/db/util/pgsession_test.go b/backend/plugin/db/util/pgsession_test.goindex d1d79ca9b3..fca7158f8f 100644--- a/backend/plugin/db/util/pgsession_test.go+++ b/backend/plugin/db/util/pgsession_test.go@@ -1,3 +1,4 @@+//nolint:revivepackage utilimport (diff --git a/go.mod b/go.modindex 107e5f4991..d42bf76b23 100644--- a/go.mod+++ b/go.mod@@ -70,6 +70,7 @@ require (github.com/lib/pq v1.12.3github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8github.com/microsoft/go-mssqldb v1.11.2+ github.com/moby/moby/api v1.56.1github.com/modelcontextprotocol/go-sdk v1.8.0github.com/nyaruka/phonenumbers/v2 v2.0.16github.com/openark/golib v0.0.0-20210531070646-355f37940af8@@ -270,7 +271,6 @@ require (github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirectgithub.com/moby/docker-image-spec v1.3.1 // indirectgithub.com/moby/go-archive v0.3.0 // indirect- github.com/moby/moby/api v1.56.1 // indirectgithub.com/moby/moby/client v0.6.2 // indirectgithub.com/moby/patternmatcher v0.6.1 // indirectgithub.com/moby/sys/sequential v0.7.0 // indirect