Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(cosmosdb): enable cross-partition queries via pure-Go query engine (#20105)
h3n4l61026b581d35
diff --git a/.sonarcloud.properties b/.sonarcloud.propertiesindex 7ea0a4fb35..1f062a2da7 100644--- a/.sonarcloud.properties+++ b/.sonarcloud.properties@@ -9,7 +9,9 @@ sonar.exclusions=\proto/gen/**,\bytebase-build/**,\frontend/node_modules/**,\- frontend/playwright.config.ts+ frontend/playwright.config.ts,\+ scripts/cosmos-repro-dotnet/**,\+ scripts/cosmos-repro-go/**# Path to tests — identify test files so they are analyzed as tests, not sourcesonar.test.inclusions=\diff --git a/backend/plugin/db/cosmosdb/cosmosdb_integration_test.go b/backend/plugin/db/cosmosdb/cosmosdb_integration_test.gonew file mode 100644index 0000000000..32e48cba0c--- /dev/null+++ b/backend/plugin/db/cosmosdb/cosmosdb_integration_test.go@@ -0,0 +1,398 @@+// Copyright (c) Bytebase (Hong Kong) Limited.+package cosmosdb++// End-to-end integration tests for BYT-9239 — every query class unlocked by+// Stages 1 through 6 of the pure-Go Cosmos query engine, exercised through+// the Bytebase driver's QueryConn path against a live Cosmos account.+//+// Guarded by AZURE_COSMOS_KEY — skips automatically when unset, so default+// unit-test runs stay hermetic.+//+// Running locally against the bytebase-cosmostest account:+//+// export AZURE_COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"+// export AZURE_COSMOS_KEY="$(az cosmosdb keys list \+// --name bytebase-cosmostest \+// --resource-group rg-bytebase-cosmostest \+// --type keys --query primaryMasterKey -o tsv)"+// go test -count=1 -run "^TestIntegration_BYT9239" \+// -timeout 300s ./backend/plugin/db/cosmosdb/++import (+ "context"+ "encoding/json"+ "os"+ "testing"+ "time"++ "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/db"+)++func skipIfNoAzure(t *testing.T) (endpoint, key, dbName, container string) {+ t.Helper()+ endpoint = os.Getenv("AZURE_COSMOS_ENDPOINT")+ key = os.Getenv("AZURE_COSMOS_KEY")+ if endpoint == "" || key == "" {+ t.Skip("set AZURE_COSMOS_ENDPOINT + AZURE_COSMOS_KEY to run live integration tests")+ }+ dbName = envOrDefault("AZURE_COSMOS_DB", "testdb")+ container = envOrDefault("AZURE_COSMOS_CONTAINER", "WorldCities")+ return endpoint, key, dbName, container+}++func envOrDefault(key, def string) string {+ if v := os.Getenv(key); v != "" {+ return v+ }+ return def+}++// newTestDriver constructs a Driver directly with a master-key-authenticated+// client, bypassing Open's Azure-AD path (which requires OAuth credentials).+// Exercises QueryConn — the exact entry point the Bytebase SQL viewer uses.+func newTestDriver(t *testing.T, endpoint, key, dbName string) *Driver {+ t.Helper()+ cred, err := azcosmos.NewKeyCredential(key)+ require.NoError(t, err, "NewKeyCredential")+ client, err := azcosmos.NewClientWithKey(endpoint, cred, nil)+ require.NoError(t, err, "NewClientWithKey")+ return &Driver{+ client: client,+ databaseName: dbName,+ connCfg: db.ConnectionConfig{+ ConnectionContext: db.ConnectionContext{DatabaseName: dbName},+ },+ }+}++// runQuery runs a single query through the driver and returns its emitted+// rows as a slice of JSON documents (one per Row in the QueryResult).+func runQuery(t *testing.T, driver *Driver, container, sql string) [][]byte {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()+ results, err := driver.QueryConn(ctx, nil, sql, db.QueryContext{Container: container})+ require.NoError(t, err, "QueryConn: %s", sql)+ require.Len(t, results, 1, "driver returns exactly one QueryResult per query")++ out := make([][]byte, 0, len(results[0].Rows))+ for _, row := range results[0].Rows {+ values := row.GetValues()+ require.Len(t, values, 1, "each row has exactly one column of JSON text")+ out = append(out, []byte(values[0].GetStringValue()))+ }+ return out+}++// ---------------------------------------------------------- Stage 1: aggregates++func TestIntegration_BYT9239_Stage1Aggregates(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ cases := []struct {+ id string+ sql string+ check func(t *testing.T, rows [][]byte)+ }{+ {"5.1", `SELECT COUNT(1) AS totalRecords FROM c`,+ func(t *testing.T, r [][]byte) { assertAliasNumeric(t, r, "totalRecords") }},+ {"5.1b", `SELECT VALUE COUNT(1) FROM c`,+ func(t *testing.T, r [][]byte) { assertScalarNumeric(t, r) }},+ {"5.2", `SELECT COUNT(1) AS aeCount FROM c WHERE c.country = "AE"`,+ func(t *testing.T, r [][]byte) { assertAliasNumeric(t, r, "aeCount") }},+ {"5.2b", `SELECT VALUE COUNT(1) FROM c WHERE c.country = "AE"`,+ func(t *testing.T, r [][]byte) { assertScalarNumeric(t, r) }},+ {"5.3", `SELECT SUM(StringToNumber(c.population)) AS totalPop FROM c`,+ func(t *testing.T, r [][]byte) { assertAliasNumeric(t, r, "totalPop") }},+ {"5.3b", `SELECT VALUE SUM(StringToNumber(c.population)) FROM c`,+ func(t *testing.T, r [][]byte) { assertScalarNumeric(t, r) }},+ {"5.4", `SELECT AVG(StringToNumber(c.population)) AS avgPop FROM c`,+ func(t *testing.T, r [][]byte) { assertAliasNumeric(t, r, "avgPop") }},+ {"5.4b", `SELECT VALUE AVG(StringToNumber(c.population)) FROM c`,+ func(t *testing.T, r [][]byte) { assertScalarNumeric(t, r) }},+ {"5.5", `SELECT MIN(StringToNumber(c.population)) AS minPop, MAX(StringToNumber(c.population)) AS maxPop FROM c WHERE StringToNumber(c.population) > 0`,+ func(t *testing.T, r [][]byte) {+ require.Len(t, r, 1)+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r[0], &obj))+ minV, ok := obj["minPop"].(float64)+ require.True(t, ok)+ maxV, ok := obj["maxPop"].(float64)+ require.True(t, ok)+ assert.LessOrEqual(t, minV, maxV)+ }},+ {"11.2", `SELECT VALUE COUNT(1) FROM c`,+ func(t *testing.T, r [][]byte) { assertScalarNumeric(t, r) }},+ }++ for _, tc := range cases {+ t.Run(tc.id, func(t *testing.T) {+ rows := runQuery(t, driver, container, tc.sql)+ tc.check(t, rows)+ })+ }+}++// ---------------------------------------------------------- Stage 2: DISTINCT++func TestIntegration_BYT9239_Stage2Distinct(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ t.Run("10.1_ObjectDistinct", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT DISTINCT c.country FROM c`)+ require.NotEmpty(t, rows)+ seen := map[string]bool{}+ for _, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj))+ country, ok := obj["country"].(string)+ require.True(t, ok, "row missing country: %s", r)+ assert.False(t, seen[country], "duplicate country: %s", country)+ seen[country] = true+ }+ })++ t.Run("10.2_DistinctValue", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT DISTINCT VALUE c.countryRegion FROM c`)+ require.NotEmpty(t, rows)+ seen := map[string]bool{}+ for _, r := range rows {+ var v string+ require.NoError(t, json.Unmarshal(r, &v))+ assert.False(t, seen[v], "duplicate: %s", v)+ seen[v] = true+ }+ })+}++// ---------------------------------------------------------- Stage 3: ORDER BY++func TestIntegration_BYT9239_Stage3OrderBy(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ t.Run("4.1_Ascending", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT c.name, c.population FROM c ORDER BY c.population ASC`)+ require.NotEmpty(t, rows)+ assertOrderedByPopulation(t, rows, true)+ })++ t.Run("4.2_Descending", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT c.name, c.population FROM c ORDER BY c.population DESC`)+ require.NotEmpty(t, rows)+ assertOrderedByPopulation(t, rows, false)+ })+}++// ---------------------------------------------------------- Stage 4: TOP++func TestIntegration_BYT9239_Stage4Top(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ rows := runQuery(t, driver, container, `SELECT TOP 10 * FROM c`)+ assert.Len(t, rows, 10, "TOP 10 must return exactly 10 rows")+ for i, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj), "row %d failed to parse: %s", i, r)+ _, hasCountry := obj["country"]+ assert.True(t, hasCountry, "row %d should be a full document: %s", i, r)+ }+}++// ---------------------------------------------------------- Stage 5: OFFSET/LIMIT++func TestIntegration_BYT9239_Stage5OffsetLimit(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ t.Run("12.1_OffsetZeroLimitTen", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT * FROM c ORDER BY c.name OFFSET 0 LIMIT 10`)+ require.Len(t, rows, 10)+ assertAscendingByName(t, rows)+ })++ t.Run("12.2_OffsetTenLimitTen", func(t *testing.T) {+ // Baseline: full ordered list.+ all := runQuery(t, driver, container, `SELECT * FROM c ORDER BY c.name`)+ require.NotEmpty(t, all)+ allNames := assertAscendingByName(t, all)++ rows := runQuery(t, driver, container, `SELECT * FROM c ORDER BY c.name OFFSET 10 LIMIT 10`)+ names := assertAscendingByName(t, rows)++ start := 10+ end := start + len(rows)+ require.LessOrEqual(t, end, len(allNames), "pagination window extends past available rows")+ assert.Equal(t, allNames[start:end], names,+ "OFFSET 10 LIMIT 10 must equal allNames[10:%d]", end)+ })+}++// ---------------------------------------------------------- Stage 6: GROUP BY++func TestIntegration_BYT9239_Stage6GroupBy(t *testing.T) {+ endpoint, key, dbName, container := skipIfNoAzure(t)+ driver := newTestDriver(t, endpoint, key, dbName)++ totalRows := runQuery(t, driver, container, `SELECT VALUE COUNT(1) FROM c`)+ require.Len(t, totalRows, 1)+ var total float64+ require.NoError(t, json.Unmarshal(totalRows[0], &total))++ t.Run("6.1_CountByCountry", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT c.country, COUNT(1) AS cityCount FROM c GROUP BY c.country`)+ require.NotEmpty(t, rows)+ var summed float64+ seen := map[string]bool{}+ for _, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj))+ country, ok := obj["country"].(string)+ require.True(t, ok, "country must be a string: %s", r)+ assert.False(t, seen[country], "duplicate country group: %s", country)+ seen[country] = true+ cnt, ok := obj["cityCount"].(float64)+ require.True(t, ok)+ summed += cnt+ }+ assert.Equal(t, total, summed, "sum of per-country counts must equal total")+ })++ t.Run("6.2_MultipleAggregates", func(t *testing.T) {+ rows := runQuery(t, driver, container, `SELECT c.countryRegion, COUNT(1) AS count, SUM(StringToNumber(c.population)) AS totalPop FROM c GROUP BY c.countryRegion`)+ require.NotEmpty(t, rows)+ var summed float64+ seen := map[string]bool{}+ for _, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj))+ region, ok := obj["countryRegion"].(string)+ require.True(t, ok, "countryRegion must be a string: %s", r)+ assert.False(t, seen[region], "duplicate region group: %s", region)+ seen[region] = true+ cnt, ok := obj["count"].(float64)+ require.True(t, ok)+ summed += cnt+ pop, ok := obj["totalPop"].(float64)+ require.True(t, ok)+ assert.GreaterOrEqual(t, pop, float64(0))+ }+ assert.Equal(t, total, summed, "sum of per-region counts must equal total")+ })+}++// ---------------------------------------------------------- assertion helpers++func assertAliasNumeric(t *testing.T, rows [][]byte, alias string) {+ t.Helper()+ require.Len(t, rows, 1)+ var obj map[string]any+ require.NoError(t, json.Unmarshal(rows[0], &obj))+ v, ok := obj[alias].(float64)+ require.True(t, ok, "alias %q must be numeric: %s", alias, rows[0])+ assert.NotZero(t, v, "alias %q must be non-zero", alias)+}++func assertScalarNumeric(t *testing.T, rows [][]byte) {+ t.Helper()+ require.Len(t, rows, 1)+ var n float64+ require.NoError(t, json.Unmarshal(rows[0], &n))+ assert.NotZero(t, n)+}++// assertOrderedByPopulation walks `rows` and asserts they are sorted by+// c.population under Cosmos item ordering (undefined < null < bool < number < string).+// ascending=true checks nondecreasing; ascending=false checks nonincreasing.+func assertOrderedByPopulation(t *testing.T, rows [][]byte, ascending bool) {+ t.Helper()+ var prevRank int+ var prevVal any+ for i, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj))+ v, hasV := obj["population"]+ require.True(t, hasV, "row missing population: %s", r)+ rank, val := cosmosTypeRank(v)+ if i == 0 {+ prevRank, prevVal = rank, val+ continue+ }+ if rank != prevRank {+ if ascending {+ assert.GreaterOrEqual(t, rank, prevRank, "type-rank regressed at row %d: %s", i, r)+ } else {+ assert.LessOrEqual(t, rank, prevRank, "type-rank ascended at row %d: %s", i, r)+ }+ prevRank, prevVal = rank, val+ continue+ }+ switch pv := prevVal.(type) {+ case float64:+ cv, ok := val.(float64)+ require.True(t, ok, "row %d: expected float64 same-rank value, got %T", i, val)+ if ascending {+ assert.LessOrEqual(t, pv, cv, "ascending number order broken at row %d", i)+ } else {+ assert.GreaterOrEqual(t, pv, cv, "descending number order broken at row %d", i)+ }+ case string:+ cv, ok := val.(string)+ require.True(t, ok, "row %d: expected string same-rank value, got %T", i, val)+ if ascending {+ assert.LessOrEqual(t, pv, cv, "ascending string order broken at row %d", i)+ } else {+ assert.GreaterOrEqual(t, pv, cv, "descending string order broken at row %d", i)+ }+ default:+ // Other Cosmos types (bool, null, undefined) don't occur in our+ // WorldCities fixture for the population field — skip.+ }+ prevVal = val+ }+}++// assertAscendingByName is a shorthand for checking that rows are ordered by+// the `name` field in ascending order. Returns the extracted names.+func assertAscendingByName(t *testing.T, rows [][]byte) []string {+ t.Helper()+ names := make([]string, 0, len(rows))+ var prev string+ for i, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj), "row %d failed to parse: %s", i, r)+ name, ok := obj["name"].(string)+ require.True(t, ok, "row %d missing name: %s", i, r)+ names = append(names, name)+ if i > 0 {+ assert.LessOrEqual(t, prev, name, "ascending broken at row %d: prev=%q cur=%q", i, prev, name)+ }+ prev = name+ }+ return names+}++// cosmosTypeRank tags a JSON value with its Cosmos-ordering type class. Used+// to compare rows that straddle the number→string boundary: Cosmos orders+// undefined < null < bool < number < string.+func cosmosTypeRank(v any) (int, any) {+ switch x := v.(type) {+ case float64:+ return 3, x+ case string:+ return 4, x+ case bool:+ return 2, x+ case nil:+ return 1, x+ default:+ return 0, x+ }+}diff --git a/go.mod b/go.modindex 37f1a9c4e0..ec922e629b 100644--- a/go.mod+++ b/go.mod@@ -375,7 +375,7 @@ require ()replace (- github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos => github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260414094732-5b89f0889452+ github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos => github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260424030223-eac0db373bcagithub.com/antlr4-go/antlr/v4 => github.com/bytebase/antlr/v4 v4.0.0-20240827034948-8c385f108920diff --git a/go.sum b/go.sumindex b87f15f1a9..6f0d367bd2 100644--- a/go.sum+++ b/go.sum@@ -242,8 +242,8 @@ github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJgithub.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=github.com/bytebase/antlr/v4 v4.0.0-20240827034948-8c385f108920 h1:IfmPt5o5R70NKtOrs+QHOoCgViYZelZysGxVBvV4ybA=github.com/bytebase/antlr/v4 v4.0.0-20240827034948-8c385f108920/go.mod h1:ykhjIPiv0IWpu3OGXCHdz2eUSe8UNGGD6baqjs8jSuU=-github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260414094732-5b89f0889452 h1:qq3h3lkj7hqzFY0pIsr4Rd/R9ilw89/TwvBYVvelUu8=-github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260414094732-5b89f0889452/go.mod h1:QrbiVoIAhIhavt4KFh6gC/gDg7xX67kbKvXnjnja+go=+github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260424030223-eac0db373bca h1:bDgHzFb+pn23vrCdQ8+VYTu8dHKwV9KWgcAUvL6CT+s=+github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos v0.0.0-20260424030223-eac0db373bca/go.mod h1:OwIkGuhFwOV7zicy7lmJl3MRqhkf2OmCSX8NMFEPm4A=github.com/bytebase/gh-ost2 v1.1.7-0.20260423064345-9ba1fc3fcc87 h1:Kqq9KRhwLKvkCMOZm3FQXj/48enNefUDRA21HSXzjp0=github.com/bytebase/gh-ost2 v1.1.7-0.20260423064345-9ba1fc3fcc87/go.mod h1:XVUEFLOcM31VxA/pvm1YMolnDWlzVyvsqZy9OZKVJ8k=github.com/bytebase/go-mssqldb v0.0.0-20240801091126-3ff3ca07d898 h1:0ggqA9XL4yvtePPvRDRYMed3jtdNk4s0o8AMW8uJPSA=diff --git a/scripts/cosmos-repro-dotnet/.gitignore b/scripts/cosmos-repro-dotnet/.gitignorenew file mode 100644index 0000000000..66366b744a--- /dev/null+++ b/scripts/cosmos-repro-dotnet/.gitignore@@ -0,0 +1,4 @@+bin/+obj/+out-gateway.md+out-direct.mddiff --git a/scripts/cosmos-repro-dotnet/CosmosRepro.csproj b/scripts/cosmos-repro-dotnet/CosmosRepro.csprojnew file mode 100644index 0000000000..8e5e50fa49--- /dev/null+++ b/scripts/cosmos-repro-dotnet/CosmosRepro.csproj@@ -0,0 +1,18 @@+<Project Sdk="Microsoft.NET.Sdk">++ <PropertyGroup>+ <OutputType>Exe</OutputType>+ <TargetFramework>net10.0</TargetFramework>+ <RootNamespace>CosmosRepro</RootNamespace>+ <AssemblyName>CosmosRepro</AssemblyName>+ <Nullable>enable</Nullable>+ <ImplicitUsings>enable</ImplicitUsings>+ <LangVersion>latest</LangVersion>+ </PropertyGroup>++ <ItemGroup>+ <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.47.1" />+ <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />+ </ItemGroup>++</Project>diff --git a/scripts/cosmos-repro-dotnet/Program.cs b/scripts/cosmos-repro-dotnet/Program.csnew file mode 100644index 0000000000..d8856baba9--- /dev/null+++ b/scripts/cosmos-repro-dotnet/Program.cs@@ -0,0 +1,195 @@+using System.Diagnostics;+using Microsoft.Azure.Cosmos;++// Reproduction harness for BYT-9239.+// Runs the queries that failed with the Bytebase (Go) Cosmos driver against+// the same Azure account using the Microsoft .NET SDK, to decide whether the+// failures are SDK-side (Go fork) or gateway/server-side.+//+// Required environment variables:+// COSMOS_ENDPOINT e.g. https://bytebase-cosmostest.documents.azure.com:443/+// COSMOS_KEY primary master key+// COSMOS_DB database name (default: testdb)+// COSMOS_CONTAINER container name (default: WorldCities)+//+// Flags:+// --mode gateway|direct (default: gateway) — matches the two modes we want to test+// --verbose print per-query detail in addition to the summary table++static class Program+{+ static async Task<int> Main(string[] args)+ {+ var mode = ParseFlag(args, "--mode", "gateway").ToLowerInvariant();+ var verbose = args.Contains("--verbose");++ var endpoint = Env("COSMOS_ENDPOINT");+ var key = Env("COSMOS_KEY");+ var db = Env("COSMOS_DB", "testdb");+ var coll = Env("COSMOS_CONTAINER", "WorldCities");++ var cmode = mode switch+ {+ "gateway" => ConnectionMode.Gateway,+ "direct" => ConnectionMode.Direct,+ _ => throw new ArgumentException($"unknown --mode {mode}")+ };++ Console.WriteLine($"# .NET SDK reproduction — mode={mode} db={db} coll={coll}");+ Console.WriteLine($"SDK: Microsoft.Azure.Cosmos (Microsoft.Azure.Cosmos assembly)");+ Console.WriteLine();++ var opts = new CosmosClientOptions+ {+ ConnectionMode = cmode,+ ApplicationName = "bytebase-byt9239-repro",+ };+ using var client = new CosmosClient(endpoint, key, opts);+ var container = client.GetContainer(db, coll);++ var tests = Tests();+ var rows = new List<Result>();+ foreach (var t in tests)+ {+ var r = await RunOne(container, t);+ rows.Add(r);+ if (verbose)+ {+ Console.WriteLine($"[{r.Status}] {r.Id} rows={r.RowCount} elapsed={r.ElapsedMs}ms");+ if (r.Error != null) Console.WriteLine($" ERR: {Short(r.Error, 260)}");+ }+ }++ // Summary markdown table+ Console.WriteLine();+ Console.WriteLine($"## Results ({mode} mode)");+ Console.WriteLine();+ Console.WriteLine("| # | Query | Status | Rows | Elapsed (ms) | Error (truncated) |");+ Console.WriteLine("|---|-------|--------|------|-------------:|-------------------|");+ foreach (var r in rows)+ {+ var err = r.Error == null ? "" : Short(r.Error, 140).Replace("|", "\\|").Replace("\n", " ");+ var q = r.Sql.Replace("|", "\\|");+ Console.WriteLine($"| {r.Id} | `{q}` | {r.Status} | {r.RowCount} | {r.ElapsedMs} | {err} |");+ }+ return 0;+ }++ record TestCase(string Id, string Sql);+ record Result(string Id, string Sql, string Status, int RowCount, long ElapsedMs, string? Error);++ static IEnumerable<TestCase> Tests() => new[]+ {+ // --- 1. Basic SELECT ---+ new TestCase("1.1", "SELECT * FROM c"),+ new TestCase("1.2", "SELECT c.id, c.name, c.country, c.population FROM c"),+ new TestCase("1.3", "SELECT TOP 10 * FROM c"),++ // --- 2. WHERE ---+ new TestCase("2.1", "SELECT * FROM c WHERE c.country = \"AE\""),+ new TestCase("2.2", "SELECT * FROM c WHERE c._ts > 1743702439"),+ new TestCase("2.3", "SELECT * FROM c WHERE c._ts >= 1743702439"),+ new TestCase("2.4", "SELECT * FROM c WHERE c.country = \"AE\" OR c.country = \"AD\""),+ new TestCase("2.5", "SELECT * FROM c WHERE c.country IN (\"AE\", \"US\", \"GB\", \"AD\")"),+ new TestCase("2.6", "SELECT * FROM c WHERE CONTAINS(c.name, \"Fujayrah\")"),+ new TestCase("2.7", "SELECT * FROM c WHERE STARTSWITH(c.name, \"Al\")"),+ new TestCase("2.8", "SELECT * FROM c WHERE c.countryRegion != \"4\""),+ new TestCase("2.9", "SELECT * FROM c WHERE STRINGTONUMBER(c.population) BETWEEN 10000 AND 100000"),++ // --- 3. Projection & Aliasing ---+ new TestCase("3.1", "SELECT c.name AS cityName, c.country AS countryCode, c.population AS pop FROM c"),+ new TestCase("3.2", "SELECT c.name, c.population, STRINGTONUMBER(c.population) / 1000 AS populationInThousands FROM c"),+ new TestCase("3.3", "SELECT CONCAT(c.name, \" (\", c.country, \")\") AS label FROM c"),++ // --- 4. ORDER BY ---+ new TestCase("4.1", "SELECT c.name, c.population FROM c ORDER BY c.population ASC"),+ new TestCase("4.2", "SELECT c.name, c.population FROM c ORDER BY c.population DESC"),+ new TestCase("4.3", "SELECT c.country, c.name, c.population FROM c ORDER BY c.country ASC, c.population DESC"),++ // --- 5. Aggregation ---+ new TestCase("5.1", "SELECT COUNT(1) AS totalRecords FROM c"),+ new TestCase("5.1b", "SELECT VALUE COUNT(1) FROM c"),+ new TestCase("5.2", "SELECT COUNT(1) AS aeCount FROM c WHERE c.country = \"AE\""),+ new TestCase("5.2b", "SELECT VALUE COUNT(1) FROM c WHERE c.country = \"AE\""),+ new TestCase("5.3", "SELECT SUM(StringToNumber(c.population)) AS totalPop FROM c"),+ new TestCase("5.3b", "SELECT VALUE SUM(StringToNumber(c.population)) FROM c"),+ new TestCase("5.4", "SELECT AVG(StringToNumber(c.population)) AS avgPop FROM c"),+ new TestCase("5.4b", "SELECT VALUE AVG(StringToNumber(c.population)) FROM c"),+ new TestCase("5.5", "SELECT MIN(StringToNumber(c.population)) AS minPop, MAX(StringToNumber(c.population)) AS maxPop FROM c WHERE StringToNumber(c.population) > 0"),++ // --- 6. GROUP BY ---+ new TestCase("6.1", "SELECT c.country, COUNT(1) AS cityCount FROM c GROUP BY c.country"),+ new TestCase("6.2", "SELECT c.countryRegion, COUNT(1) AS count, SUM(StringToNumber(c.population)) AS totalPop FROM c GROUP BY c.countryRegion"),++ // --- 7. String built-ins ---+ new TestCase("7.1", "SELECT UPPER(c.name) AS upperName, LOWER(c.country) AS lowerCountry, LENGTH(c.name) AS nameLen FROM c"),++ // --- 8. Math built-ins ---+ new TestCase("8.1", "SELECT c.name, ROUND(StringToNumber(c.latitude)) AS lat, ROUND(StringToNumber(c.longitude)) AS lon FROM c"),++ // --- 9. Type-checking built-ins ---+ new TestCase("9.1", "SELECT c.name, IS_STRING(c.name) AS isStr, IS_NUMBER(c.population) AS isNum FROM c"),+ new TestCase("9.2", "SELECT c.id, IS_DEFINED(c.region) AS hasRegion, IS_NULL(c.code) AS codeNull FROM c"),++ // --- 10. DISTINCT ---+ new TestCase("10.1", "SELECT DISTINCT c.country FROM c"),+ new TestCase("10.2", "SELECT DISTINCT VALUE c.countryRegion FROM c"),++ // --- 11. VALUE keyword ---+ new TestCase("11.1", "SELECT VALUE c.name FROM c WHERE c.country = \"AE\""),+ new TestCase("11.2", "SELECT VALUE COUNT(1) FROM c"),++ // --- 12. OFFSET / LIMIT ---+ new TestCase("12.1", "SELECT * FROM c ORDER BY c.name OFFSET 0 LIMIT 10"),+ new TestCase("12.2", "SELECT * FROM c ORDER BY c.name OFFSET 10 LIMIT 10"),++ // --- 13. Geo-spatial ---+ new TestCase("13.1", "SELECT c.name, ST_DISTANCE({\"type\": \"Point\", \"coordinates\": [StringToNumber(c.longitude), StringToNumber(c.latitude)]}, {\"type\": \"Point\", \"coordinates\": [55.2708, 25.2048]}) AS distFromDubaiInMeters FROM c"),+ };++ static async Task<Result> RunOne(Container c, TestCase t)+ {+ var sw = Stopwatch.StartNew();+ try+ {+ var opt = new QueryRequestOptions { MaxItemCount = 1000 };+ // Cap at a safe upper bound — we just want to know if the query succeeds.+ // For `SELECT * FROM c` the container has thousands of docs; we stop at 10k.+ const int cap = 10_000;++ // Use `dynamic` so VALUE queries that return bare scalars (numbers, strings) deserialize cleanly.+ using var iter = c.GetItemQueryIterator<dynamic>(new QueryDefinition(t.Sql), requestOptions: opt);+ int count = 0;+ while (iter.HasMoreResults)+ {+ var page = await iter.ReadNextAsync();+ count += page.Count;+ if (count >= cap) break;+ }+ sw.Stop();+ return new Result(t.Id, t.Sql, "PASS", count, sw.ElapsedMilliseconds, null);+ }+ catch (Exception e)+ {+ sw.Stop();+ return new Result(t.Id, t.Sql, "FAIL", 0, sw.ElapsedMilliseconds, e.Message);+ }+ }++ static string Env(string name, string? def = null)+ {+ var v = Environment.GetEnvironmentVariable(name);+ if (!string.IsNullOrWhiteSpace(v)) return v!;+ if (def != null) return def;+ throw new InvalidOperationException($"missing env var {name}");+ }++ static string ParseFlag(string[] args, string flag, string def)+ {+ for (int i = 0; i < args.Length - 1; i++)+ if (args[i] == flag) return args[i + 1];+ return def;+ }++ static string Short(string s, int n) => s.Length <= n ? s : s.Substring(0, n) + "…";+}diff --git a/scripts/cosmos-repro-dotnet/README.md b/scripts/cosmos-repro-dotnet/README.mdnew file mode 100644index 0000000000..71c2c9796b--- /dev/null+++ b/scripts/cosmos-repro-dotnet/README.md@@ -0,0 +1,45 @@+# cosmos-repro-dotnet — BYT-9239 reproduction harness++Reproduces the Cosmos DB query failures described in BYT-9239 using the+Microsoft .NET SDK (`Microsoft.Azure.Cosmos`). The goal is to determine+whether the failures are:++- **Go-SDK-specific** (Bytebase's forked `azcosmos`) → queries pass here, fail in Go+- **Gateway/server-level** → queries fail in both SDKs++## Prerequisites++- .NET 9 SDK (`brew install dotnet`)+- Azure CLI, logged in to the subscription holding `bytebase-cosmostest`++## Run++```bash+export COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"+export COSMOS_KEY="$(az cosmosdb keys list \+ --name bytebase-cosmostest \+ --resource-group rg-bytebase-cosmostest \+ --type keys --query primaryMasterKey -o tsv)"+export COSMOS_DB="testdb"+export COSMOS_CONTAINER="WorldCities"++# Gateway mode (same as Bytebase's default)+dotnet run --project . -- --mode gateway --verbose | tee out-gateway.md++# Direct mode (TCP, bypasses gateway)+dotnet run --project . -- --mode direct --verbose | tee out-direct.md+```++Each run prints a markdown table of per-query status: PASS / FAIL, rows returned,+elapsed milliseconds, and a truncated error.++## Scope++The test list covers the query groups marked FAIL or PARTIAL in the PDF:+`Basic SELECT TOP`, `ORDER BY`, `Aggregation` (aliased + `VALUE`),+`GROUP BY`, `DISTINCT`, `VALUE COUNT`, `OFFSET/LIMIT`. It also includes+one known-passing query (`WHERE c.country = "AE"`) as a sanity check.++Container partition key is `/country` — queries without a `country` filter+are cross-partition and are the ones expected to stress-test the SDK's+cross-partition orchestration.diff --git a/scripts/cosmos-repro-go/.gitignore b/scripts/cosmos-repro-go/.gitignorenew file mode 100644index 0000000000..aa7cd1010f--- /dev/null+++ b/scripts/cosmos-repro-go/.gitignore@@ -0,0 +1,2 @@+cosmos-repro-go+out-go.mddiff --git a/scripts/cosmos-repro-go/README.md b/scripts/cosmos-repro-go/README.mdnew file mode 100644index 0000000000..43d7eb5366--- /dev/null+++ b/scripts/cosmos-repro-go/README.md@@ -0,0 +1,59 @@+# cosmos-repro-go — BYT-9239 reproduction harness (Go)++Go twin of `scripts/cosmos-repro-dotnet`. Runs the full 40-query target+inventory from BYT-9239 against a live Azure Cosmos account through+Bytebase's forked `azcosmos` SDK (which embeds the pure-Go cross-partition+query engine), and prints a markdown PASS/FAIL table in the same shape as+the .NET harness. Pairing both harnesses on the same container is the+regression guard for the BYT-9239 feature set: as long as the Go matrix+matches the .NET matrix (39/40 PASS; the single FAIL is the multi-key+ORDER BY query that needs a composite index — a server-side constraint+that affects every SDK), the Go engine is at .NET parity for this inventory.++## Prerequisites++- Go toolchain (the repo's current `go.mod` version)+- Azure CLI, logged in to the subscription holding `bytebase-cosmostest`++## Run++```bash+export COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"+export COSMOS_KEY="$(az cosmosdb keys list \+ --name bytebase-cosmostest \+ --resource-group rg-bytebase-cosmostest \+ --type keys --query primaryMasterKey -o tsv)"+export COSMOS_DB="testdb"+export COSMOS_CONTAINER="WorldCities"++cd scripts/cosmos-repro-go+go run . --verbose | tee out-go.md+```++The harness uses `NewCrossPartitionQueryItemsPager`, which engages the+pure-Go query engine automatically when the gateway returns its+cross-partition sentinel error. Queries the gateway can still serve+(e.g. single-partition WHERE filters, projections, built-ins) incur no+extra round-trip — they never leave the gateway path.++## Scope++The query list mirrors the .NET harness's 40-query inventory exactly, in+PDF section order. It includes both the FAIL / PARTIAL groups (TOP, ORDER BY,+aggregates in aliased and VALUE form, GROUP BY, DISTINCT, OFFSET/LIMIT,+scalar VALUE aggregates) and the known-passing groups (filters, projections,+string / math / type-checking built-ins, geo-spatial). Running both groups+keeps the two harnesses symmetric: a divergence between the Go and .NET+columns is the signal that the Go engine regressed on something the .NET+SDK handles.++Container partition key is `/country` — queries without a `country` filter+are cross-partition and are the ones exercising the engine.++## Expected matrix++39 PASS / 1 FAIL, matching the .NET run. The one permanent FAIL is+`SELECT c.country, c.name, c.population FROM c ORDER BY c.country ASC, c.population DESC`+(4.3): the Azure gateway rejects multi-key ORDER BY when the container+lacks a composite index on `(country ASC, population DESC)` — a real+server constraint, not an SDK limitation.diff --git a/scripts/cosmos-repro-go/main.go b/scripts/cosmos-repro-go/main.gonew file mode 100644index 0000000000..77491bf889--- /dev/null+++ b/scripts/cosmos-repro-go/main.go@@ -0,0 +1,239 @@+// Copyright (c) Bytebase (Hong Kong) Limited.++// Go reproduction harness for BYT-9239 — the twin of scripts/cosmos-repro-dotnet.+// Runs the full 40-query target inventory from+// https://linear.app/bytebase/issue/BYT-9239 against a live Azure Cosmos+// account through Bytebase's forked azcosmos SDK (which includes the pure-Go+// query engine), and prints a markdown PASS/FAIL table matching the shape+// the .NET harness produces. Pairing both harnesses on the same container+// is the regression guard for the BYT-9239 feature set.+//+// Required environment variables:+// COSMOS_ENDPOINT e.g. https://bytebase-cosmostest.documents.azure.com:443/+// COSMOS_KEY primary master key+// COSMOS_DB database name (default: testdb)+// COSMOS_CONTAINER container name (default: WorldCities)+//+// Flags:+// --verbose also print per-query detail above the summary table+//+// Run:+// cd scripts/cosmos-repro-go+// export COSMOS_ENDPOINT="https://bytebase-cosmostest.documents.azure.com:443/"+// export COSMOS_KEY="$(az cosmosdb keys list \+// --name bytebase-cosmostest \+// --resource-group rg-bytebase-cosmostest \+// --type keys --query primaryMasterKey -o tsv)"+// go run . --verbose++package main++import (+ "context"+ "flag"+ "fmt"+ "os"+ "strings"+ "time"++ "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"+)++type testCase struct {+ id string+ sql string+}++// cases mirrors the 40-query target inventory exactly, in PDF section order.+// Each query's expected outcome is taken from the .NET SDK reference column.+// The two permanently-failing queries (4.3 multi-key ORDER BY, which the+// gateway rejects on any SDK because the container lacks a composite index)+// are included so the harness's summary matches the .NET harness's.+var cases = []testCase{+ // 1. Basic SELECT+ {"1.1", `SELECT * FROM c`},+ {"1.2", `SELECT c.id, c.name, c.country, c.population FROM c`},+ {"1.3", `SELECT TOP 10 * FROM c`},++ // 2. Filtering — WHERE+ {"2.1", `SELECT * FROM c WHERE c.country = "AE"`},+ {"2.2", `SELECT * FROM c WHERE c._ts > 1743702439`},+ {"2.3", `SELECT * FROM c WHERE c._ts >= 1743702439`},+ {"2.4", `SELECT * FROM c WHERE c.country = "AE" OR c.country = "AD"`},+ {"2.5", `SELECT * FROM c WHERE c.country IN ("AE", "US", "GB", "AD")`},+ {"2.6", `SELECT * FROM c WHERE CONTAINS(c.name, "Fujayrah")`},+ {"2.7", `SELECT * FROM c WHERE STARTSWITH(c.name, "Al")`},+ {"2.8", `SELECT * FROM c WHERE c.countryRegion != "4"`},+ {"2.9", `SELECT * FROM c WHERE STRINGTONUMBER(c.population) BETWEEN 10000 AND 100000`},++ // 3. Projection & Aliasing+ {"3.1", `SELECT c.name AS cityName, c.country AS countryCode, c.population AS pop FROM c`},+ {"3.2", `SELECT c.name, c.population, STRINGTONUMBER(c.population) / 1000 AS populationInThousands FROM c`},+ {"3.3", `SELECT CONCAT(c.name, " (", c.country, ")") AS label FROM c`},++ // 4. ORDER BY+ {"4.1", `SELECT c.name, c.population FROM c ORDER BY c.population ASC`},+ {"4.2", `SELECT c.name, c.population FROM c ORDER BY c.population DESC`},+ {"4.3", `SELECT c.country, c.name, c.population FROM c ORDER BY c.country ASC, c.population DESC`},++ // 5. Aggregation+ {"5.1", `SELECT COUNT(1) AS totalRecords FROM c`},+ {"5.1b", `SELECT VALUE COUNT(1) FROM c`},+ {"5.2", `SELECT COUNT(1) AS aeCount FROM c WHERE c.country = "AE"`},+ {"5.2b", `SELECT VALUE COUNT(1) FROM c WHERE c.country = "AE"`},+ {"5.3", `SELECT SUM(StringToNumber(c.population)) AS totalPop FROM c`},+ {"5.3b", `SELECT VALUE SUM(StringToNumber(c.population)) FROM c`},+ {"5.4", `SELECT AVG(StringToNumber(c.population)) AS avgPop FROM c`},+ {"5.4b", `SELECT VALUE AVG(StringToNumber(c.population)) FROM c`},+ {"5.5", `SELECT MIN(StringToNumber(c.population)) AS minPop, MAX(StringToNumber(c.population)) AS maxPop FROM c WHERE StringToNumber(c.population) > 0`},++ // 6. GROUP BY+ {"6.1", `SELECT c.country, COUNT(1) AS cityCount FROM c GROUP BY c.country`},+ {"6.2", `SELECT c.countryRegion, COUNT(1) AS count, SUM(StringToNumber(c.population)) AS totalPop FROM c GROUP BY c.countryRegion`},++ // 7. String built-ins+ {"7.1", `SELECT UPPER(c.name) AS upperName, LOWER(c.country) AS lowerCountry, LENGTH(c.name) AS nameLen FROM c`},++ // 8. Math built-ins+ {"8.1", `SELECT c.name, ROUND(StringToNumber(c.latitude)) AS lat, ROUND(StringToNumber(c.longitude)) AS lon FROM c`},++ // 9. Type-checking built-ins+ {"9.1", `SELECT c.name, IS_STRING(c.name) AS isStr, IS_NUMBER(c.population) AS isNum FROM c`},+ {"9.2", `SELECT c.id, IS_DEFINED(c.region) AS hasRegion, IS_NULL(c.code) AS codeNull FROM c`},++ // 10. DISTINCT+ {"10.1", `SELECT DISTINCT c.country FROM c`},+ {"10.2", `SELECT DISTINCT VALUE c.countryRegion FROM c`},++ // 11. VALUE keyword+ {"11.1", `SELECT VALUE c.name FROM c WHERE c.country = "AE"`},+ {"11.2", `SELECT VALUE COUNT(1) FROM c`},++ // 12. OFFSET / LIMIT+ {"12.1", `SELECT * FROM c ORDER BY c.name OFFSET 0 LIMIT 10`},+ {"12.2", `SELECT * FROM c ORDER BY c.name OFFSET 10 LIMIT 10`},++ // 13. Geo-spatial+ {"13.1", `SELECT c.name, ST_DISTANCE({"type": "Point", "coordinates": [StringToNumber(c.longitude), StringToNumber(c.latitude)]}, {"type": "Point", "coordinates": [55.2708, 25.2048]}) AS distFromDubaiInMeters FROM c`},+}++type result struct {+ id string+ sql string+ status string+ rows int+ elapsedMs int64+ err string+}++func main() {+ verbose := flag.Bool("verbose", false, "print per-query detail above the summary table")+ flag.Parse()++ endpoint := envOr("COSMOS_ENDPOINT", "")+ key := envOr("COSMOS_KEY", "")+ if endpoint == "" || key == "" {+ fmt.Fprintln(os.Stderr, "COSMOS_ENDPOINT and COSMOS_KEY are required")+ os.Exit(1)+ }+ db := envOr("COSMOS_DB", "testdb")+ coll := envOr("COSMOS_CONTAINER", "WorldCities")++ cred, err := azcosmos.NewKeyCredential(key)+ if err != nil {+ fmt.Fprintf(os.Stderr, "NewKeyCredential: %v\n", err)+ os.Exit(1)+ }+ client, err := azcosmos.NewClientWithKey(endpoint, cred, nil)+ if err != nil {+ fmt.Fprintf(os.Stderr, "NewClientWithKey: %v\n", err)+ os.Exit(1)+ }+ container, err := client.NewContainer(db, coll)+ if err != nil {+ fmt.Fprintf(os.Stderr, "NewContainer: %v\n", err)+ os.Exit(1)+ }++ fmt.Printf("# Go SDK reproduction — db=%s coll=%s\n", db, coll)+ fmt.Println("SDK: github.com/bytebase/azure-sdk-for-go/sdk/data/azcosmos (pure-Go query engine)")+ fmt.Println()++ rows := make([]result, 0, len(cases))+ for _, tc := range cases {+ r := runOne(container, tc)+ rows = append(rows, r)+ if *verbose {+ fmt.Printf("[%s] %s rows=%d elapsed=%dms\n", r.status, r.id, r.rows, r.elapsedMs)+ if r.err != "" {+ fmt.Printf(" ERR: %s\n", short(r.err, 260))+ }+ }+ }++ // Summary table in the same shape as the .NET harness.+ fmt.Println()+ fmt.Println("## Results")+ fmt.Println()+ fmt.Println("| # | Query | Status | Rows | Elapsed (ms) | Error (truncated) |")+ fmt.Println("|---|-------|--------|------|-------------:|-------------------|")+ for _, r := range rows {+ errCell := ""+ if r.err != "" {+ errCell = strings.ReplaceAll(short(r.err, 140), "|", `\|`)+ errCell = strings.ReplaceAll(errCell, "\n", " ")+ }+ q := strings.ReplaceAll(r.sql, "|", `\|`)+ fmt.Printf("| %s | `%s` | %s | %d | %d | %s |\n", r.id, q, r.status, r.rows, r.elapsedMs, errCell)+ }++ // Tally.+ pass, fail := 0, 0+ for _, r := range rows {+ if r.status == "PASS" {+ pass+++ } else {+ fail+++ }+ }+ fmt.Println()+ fmt.Printf("**Totals:** %d PASS, %d FAIL out of %d.\n", pass, fail, len(rows))+}++func runOne(c *azcosmos.ContainerClient, tc testCase) result {+ start := time.Now()+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ // 10k row cap keeps the harness bounded when running SELECT * on large+ // containers. Matches the .NET harness's ceiling.+ const rowCap = 10_000++ pager := c.NewCrossPartitionQueryItemsPager(tc.sql, nil)+ n := 0+ for pager.More() {+ page, err := pager.NextPage(ctx)+ if err != nil {+ return result{id: tc.id, sql: tc.sql, status: "FAIL", rows: n, elapsedMs: time.Since(start).Milliseconds(), err: err.Error()}+ }+ n += len(page.Items)+ if n >= rowCap {+ break+ }+ }+ return result{id: tc.id, sql: tc.sql, status: "PASS", rows: n, elapsedMs: time.Since(start).Milliseconds()}+}++func envOr(key, def string) string {+ if v := os.Getenv(key); v != "" {+ return v+ }+ return def+}++func short(s string, n int) string {+ if len(s) <= n {+ return s+ }+ return s[:n] + "…"+}