Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(cosmosdb): implement the query-plan operators the engine composed wrongly (#21895)
Danny Xu219bbdcbfc57
diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.goindex af21aa9fb9..2a14ad5f3a 100644--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go@@ -998,3 +998,280 @@ func TestCrossPartitionQueryLimit(t *testing.T) {assert.Len(t, results[0].Rows, 7)assert.EqualValues(t, 7, results[0].RowsCount)}++// ------------------------------------ composed operators, local emulator+//+// The query classes in BYT-10397: each composes two operators the gateway+// reports in one plan, and the engine used to honour only one of them. Every+// case here runs on the multi-page fixture, because two of them (TOP with+// ORDER BY, and DISTINCT with a LIMIT) answer correctly on a single physical+// partition however the engine is broken.++// pagedQuery runs a query against the multi-page fixture and returns its rows+// as raw JSON, in the order the engine emitted them.+func pagedQuery(t *testing.T, container *azcosmos.ContainerClient, sql string) []string {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)+ defer cancel()+ items, err := QueryEveryPartition(ctx, container, sql, nil, 0)+ require.NoError(t, err, "QueryEveryPartition: %s", sql)+ rows := make([]string, 0, len(items))+ for _, item := range items {+ rows = append(rows, string(item))+ }+ return rows+}++// pagedBucketValues reads the bucket out of every row, for both spellings of+// a projection: `{"bucket":"big-1"}` and the VALUE form's bare `"big-1"`.+func pagedBucketValues(t *testing.T, rows []string) []string {+ t.Helper()+ out := make([]string, 0, len(rows))+ for _, row := range rows {+ var object struct {+ Bucket *string `json:"bucket"`+ }+ if err := json.Unmarshal([]byte(row), &object); err == nil && object.Bucket != nil {+ out = append(out, *object.Bucket)+ continue+ }+ var scalar string+ require.NoError(t, json.Unmarshal([]byte(row), &scalar), "row: %s", row)+ out = append(out, scalar)+ }+ return out+}++// sortedPagedBuckets is the fixture's partition key values in ascending order.+func sortedPagedBuckets() []string {+ buckets := append(append([]string(nil), pagedBigBuckets...), pagedSmallBuckets...)+ slices.Sort(buckets)+ return buckets+}++// pagedBucketRows is how many rows each partition key value holds.+func pagedBucketRows(bucket string) int {+ if slices.Contains(pagedBigBuckets, bucket) {+ return pagedBigRows+ }+ return pagedSmallRows+}++// TestCrossPartitionDistinctComposed covers DISTINCT alongside an ORDER BY or+// a page. The service reports distinctType "Unordered" for the object form+// even under ORDER BY, so the engine dropped the DISTINCT and returned every+// row in the container; a paged DISTINCT ran the user's own query per range,+// so each range applied the LIMIT to itself before anything was deduplicated.+func TestCrossPartitionDistinctComposed(t *testing.T) {+ container := pagedFixture(t)+ buckets := sortedPagedBuckets()++ t.Run("order_by_object_form", func(t *testing.T) {+ assert.Equal(t, buckets, pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket`)))+ })++ t.Run("order_by_value_form", func(t *testing.T) {+ assert.Equal(t, buckets, pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT VALUE c.bucket FROM c ORDER BY c.bucket`)))+ })++ t.Run("order_by_descending", func(t *testing.T) {+ want := sortedPagedBuckets()+ slices.Reverse(want)+ assert.Equal(t, want, pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket DESC`)))+ })++ // An ordered DISTINCT pins the page against exact values; the unordered+ // forms below can only be held to the count, since Cosmos defines no+ // order for them.+ t.Run("order_by_with_a_page", func(t *testing.T) {+ assert.Equal(t, buckets[2:5], pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket OFFSET 2 LIMIT 3`)))+ })++ t.Run("limit_value_form", func(t *testing.T) {+ got := pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT VALUE c.bucket FROM c OFFSET 0 LIMIT 3`))+ assert.Len(t, got, 3)+ assert.Subset(t, buckets, got)+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(got))), 3, "values must be distinct")+ })++ t.Run("offset_limit_object_form", func(t *testing.T) {+ got := pagedBucketValues(t,+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c OFFSET 2 LIMIT 3`))+ assert.Len(t, got, 3)+ assert.Subset(t, buckets, got)+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(got))), 3, "values must be distinct")+ })+}++// TestCrossPartitionTopWithOrderBy covers TOP alongside an ORDER BY. The+// rewritten per-range query carries the TOP, so the merge saw n rows from+// every range and emitted all of them: n x partitions rows, in order.+func TestCrossPartitionTopWithOrderBy(t *testing.T) {+ container := pagedFixture(t)+ rows := pagedRows()++ t.Run("ascending", func(t *testing.T) {+ assert.Equal(t, ascendingSeqs(0, 5),+ pagedSeqs(t, container, `SELECT TOP 5 c.seq FROM c ORDER BY c.seq ASC`))+ })++ t.Run("descending", func(t *testing.T) {+ want := ascendingSeqs(rows-5, rows)+ slices.Reverse(want)+ assert.Equal(t, want, pagedSeqs(t, container, `SELECT TOP 5 c.seq FROM c ORDER BY c.seq DESC`))+ })++ // A TOP past the gateway's page cap has to keep reading the ranges after+ // the cap, and still stop at n.+ t.Run("above_the_page_cap", func(t *testing.T) {+ assert.Equal(t, ascendingSeqs(0, pagedPageSize+25),+ pagedSeqs(t, container, fmt.Sprintf(+ `SELECT TOP %d c.seq FROM c ORDER BY c.seq ASC`, pagedPageSize+25)))+ })+}++// TestCrossPartitionGroupByOrder covers the order GROUP BY rows come back in.+// The pipeline emitted them straight out of a Go map, so the same query+// answered in a different order every run, and a GROUP BY the user ordered+// reaches the engine as a plan with no order fields at all — the ORDER BY+// survives only inside the rewritten query.+func TestCrossPartitionGroupByOrder(t *testing.T) {+ container := pagedFixture(t)++ type group struct {+ Bucket string `json:"bucket"`+ N int `json:"n"`+ }+ groupsOf := func(t *testing.T, rows []string) []group {+ t.Helper()+ out := make([]group, 0, len(rows))+ for _, row := range rows {+ var g group+ require.NoError(t, json.Unmarshal([]byte(row), &g), "row: %s", row)+ out = append(out, g)+ }+ return out+ }+ want := make([]group, 0, len(sortedPagedBuckets()))+ for _, bucket := range sortedPagedBuckets() {+ want = append(want, group{Bucket: bucket, N: pagedBucketRows(bucket)})+ }++ t.Run("order_by_the_group_key", func(t *testing.T) {+ assert.Equal(t, want, groupsOf(t, pagedQuery(t, container,+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket`)))+ })++ t.Run("order_by_descending", func(t *testing.T) {+ reversed := slices.Clone(want)+ slices.Reverse(reversed)+ assert.Equal(t, reversed, groupsOf(t, pagedQuery(t, container,+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket DESC`)))+ })++ // Without an ORDER BY the order is Cosmos's to choose, but it must be the+ // same on every run.+ t.Run("unordered_is_stable_across_runs", func(t *testing.T) {+ const sql = `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`+ first := groupsOf(t, pagedQuery(t, container, sql))+ assert.ElementsMatch(t, want, first)+ for range 2 {+ assert.Equal(t, first, groupsOf(t, pagedQuery(t, container, sql)))+ }+ })+}++// TestCrossPartitionEmptyAggregates covers an aggregate whose filter matches+// no row. MIN and MAX failed the query outright, because an empty partition's+// partial is an absent one; the aliased form answered `{"m":null}` where the+// gateway omits the alias. COUNT and SUM really are 0 over an empty set and+// are here as controls.+func TestCrossPartitionEmptyAggregates(t *testing.T) {+ container := pagedFixture(t)+ const noRows = `WHERE c.seq > 99999999`++ for _, tc := range []struct {+ name string+ sql string+ want []string+ }{+ {"min_value", `SELECT VALUE MIN(c.seq) FROM c ` + noRows, nil},+ {"max_value", `SELECT VALUE MAX(c.seq) FROM c ` + noRows, nil},+ {"avg_value", `SELECT VALUE AVG(c.seq) FROM c ` + noRows, nil},+ {"min_aliased", `SELECT MIN(c.seq) AS m FROM c ` + noRows, []string{`{}`}},+ {"min_max_aliased", `SELECT MIN(c.seq) AS lo, MAX(c.seq) AS hi FROM c ` + noRows, []string{`{}`}},+ {"count_value", `SELECT VALUE COUNT(1) FROM c ` + noRows, []string{`0`}},+ {"count_aliased", `SELECT COUNT(1) AS n FROM c ` + noRows, []string{`{"n":0}`}},+ {"sum_value", `SELECT VALUE SUM(c.seq) FROM c ` + noRows, []string{`0`}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got := pagedQuery(t, container, tc.sql)+ require.Len(t, got, len(tc.want))+ for i := range tc.want {+ assert.JSONEq(t, tc.want[i], got[i])+ }+ })+ }++ // The same MIN and MAX over rows that do match: the VALUE form reads its+ // partial from item2, which is where the count lives.+ t.Run("min_max_over_rows_that_match", func(t *testing.T) {+ assert.Equal(t, []string{`0`}, pagedQuery(t, container, `SELECT VALUE MIN(c.seq) FROM c`))+ assert.Equal(t, []string{fmt.Sprint(pagedRows() - 1)},+ pagedQuery(t, container, `SELECT VALUE MAX(c.seq) FROM c`))+ })+}++// TestCrossPartitionOffsetLimitWithoutOrderBy covers ordinary result paging,+// which the engine rejected although it advertises OffsetAndLimit. Cosmos+// defines no order for it, so only the count and the rows' provenance hold.+func TestCrossPartitionOffsetLimitWithoutOrderBy(t *testing.T) {+ container := pagedFixture(t)++ seqs := pagedSeqs(t, container, `SELECT c.seq FROM c OFFSET 2 LIMIT 3`)+ require.Len(t, seqs, 3)+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(seqs))), 3, "rows must not repeat")+ for _, seq := range seqs {+ assert.Less(t, seq, pagedRows())+ assert.GreaterOrEqual(t, seq, 0)+ }++ // A window wider than the gateway's page cap reads several pages of a+ // range before it is full.+ wide := pagedSeqs(t, container, fmt.Sprintf(`SELECT c.seq FROM c OFFSET 0 LIMIT %d`, pagedPageSize+25))+ assert.Len(t, wide, pagedPageSize+25)+}++// TestCrossPartitionAggregateWithOffsetLimit covers a page applied to an+// aggregate. The query produces exactly one row, so OFFSET 1 skips past it and+// the gateway answers with nothing; the engine returned the row regardless.+func TestCrossPartitionAggregateWithOffsetLimit(t *testing.T) {+ container := pagedFixture(t)++ assert.Empty(t, pagedQuery(t, container, `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`))+ assert.Equal(t, []string{fmt.Sprint(pagedRows())},+ pagedQuery(t, container, `SELECT VALUE COUNT(1) FROM c OFFSET 0 LIMIT 1`))+ assert.Empty(t, pagedQuery(t, container, `SELECT COUNT(1) AS n FROM c OFFSET 1 LIMIT 1`))+}++// TestCrossPartitionDistinctWithOrderByThroughTheDriver runs the worst of+// these through QueryConn, the entry point the SQL editor uses, so the SQL+// validator is in the path too.+func TestCrossPartitionDistinctWithOrderByThroughTheDriver(t *testing.T) {+ driver := emulatorFixture(t)+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT c.country FROM c ORDER BY c.country`)+ got := make([]string, 0, len(rows))+ for _, row := range rows {+ var object struct {+ Country string `json:"country"`+ }+ require.NoError(t, json.Unmarshal(row, &object))+ got = append(got, object.Country)+ }+ assert.Equal(t, slices.Sorted(slices.Values(crossPartitionCountries)), got)+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg.goindex 2eb8e0ea63..044b23aac3 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go@@ -2,6 +2,7 @@ package gonativeimport ("encoding/json"+ "fmt""strconv""github.com/pkg/errors"@@ -19,8 +20,11 @@ import (// Max — {"max": <value>, "count": <int>}//// finalize returns the final scalar as a json.RawMessage so the pipeline can-// emit it without re-marshaling. Returns an empty value (JSON null) when no-// contributing rows were observed (e.g. Avg over an empty set).+// emit it without re-marshaling. It returns a nil message when the aggregate+// has no value: Cosmos reports MIN, MAX and AVG over an empty set as+// undefined, which the gateway renders as no row at all in the VALUE form and+// as an absent alias in the aliased form. COUNT and SUM over an empty set are+// 0, and the gateway sends that 0 as a partial, so they never finalize nil.type aggregator interface {feedPartial(raw json.RawMessage) errorfinalize() (json.RawMessage, error)@@ -47,6 +51,45 @@ func newAggregator(kind string) aggregator {return nil}+// aggregatePartial returns the partial feedPartial expects for one aggregate,+// given the gateway's {item, item2} document entry. MIN and MAX read item2:+// only it reports how many rows the partition contributed, which is how a+// partition that matched nothing contributes nothing. item2 is absent where+// the gateway short-circuited to a single-partition shape, and item is then+// the value itself. Every other kind carries its partial in item alone.+//+// Returns nil when the gateway left the partial out, which it does for an+// aggregate over a partition that matched no row.+func aggregatePartial(kind string, it aggItem) json.RawMessage {+ switch kind {+ case "Min", "Max":+ if len(it.Item2) > 0 {+ return it.Item2+ }+ if len(it.Item) == 0 {+ return nil+ }+ key := "min"+ if kind == "Max" {+ key = "max"+ }+ return fmt.Appendf(nil, `{%q:%s,"count":1}`, key, it.Item)+ default:+ return it.Item+ }+}++// feedAggregatePartial folds one partition document's partial into a. A+// partial the gateway left out is not an error: the partition had no row to+// contribute.+func feedAggregatePartial(a aggregator, kind string, it aggItem) error {+ partial := aggregatePartial(kind, it)+ if len(partial) == 0 {+ return nil+ }+ return a.feedPartial(partial)+}+// -----------------------------------------------------------------------------// Count@@ -87,7 +130,7 @@ func (a *sumAgg) feedPartial(raw json.RawMessage) error {func (a *sumAgg) finalize() (json.RawMessage, error) {if !a.seen {- return json.RawMessage(`null`), nil+ return nil, nil}return numberRawMessage(a.total), nil}@@ -129,7 +172,7 @@ func (a *minAgg) feedPartial(raw json.RawMessage) error {func (a *minAgg) finalize() (json.RawMessage, error) {if a.best == nil {- return json.RawMessage(`null`), nil+ return nil, nil}return a.best, nil}@@ -170,7 +213,7 @@ func (a *maxAgg) feedPartial(raw json.RawMessage) error {func (a *maxAgg) finalize() (json.RawMessage, error) {if a.best == nil {- return json.RawMessage(`null`), nil+ return nil, nil}return a.best, nil}@@ -200,9 +243,7 @@ func (a *avgAgg) feedPartial(raw json.RawMessage) error {func (a *avgAgg) finalize() (json.RawMessage, error) {if a.count == 0 {- // Matches Cosmos semantics: AVG over an empty set is undefined;- // gateway serializes this as JSON null for cross-partition results.- return json.RawMessage(`null`), nil+ return nil, nil}return numberRawMessage(a.sum / float64(a.count)), nil}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.goindex 526f2691b6..a08a7ddfdb 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go@@ -51,9 +51,9 @@ func TestCountAgg_SumsAcrossPartitions(t *testing.T) {// ---------------------------------------------------------- Sum-func TestSumAgg_EmptyFinalizesToNull(t *testing.T) {+func TestSumAgg_EmptyFinalizesToUndefined(t *testing.T) {out := feedFinal(t, newAggregator("Sum"))- assert.Equal(t, json.RawMessage(`null`), out)+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")}func TestSumAgg_SumsAcrossPartitions(t *testing.T) {@@ -64,9 +64,9 @@ func TestSumAgg_SumsAcrossPartitions(t *testing.T) {// ---------------------------------------------------------- Avg-func TestAvgAgg_EmptyFinalizesToNull(t *testing.T) {+func TestAvgAgg_EmptyFinalizesToUndefined(t *testing.T) {out := feedFinal(t, newAggregator("Avg"))- assert.Equal(t, json.RawMessage(`null`), out)+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")}func TestAvgAgg_SinglePartition(t *testing.T) {@@ -88,11 +88,43 @@ func TestAvgAgg_ZeroCountPartialContributesNothing(t *testing.T) {assert.Equal(t, json.RawMessage(`2`), out)}+// ---------------------------------------------------------- partial routing++// A MIN or MAX partial comes from item2, the only one that reports how many+// rows the partition contributed; a partition that matched none contributes+// nothing at all.+func TestAggregatePartial_MinMaxPreferItem2(t *testing.T) {+ both := aggItem{Item: json.RawMessage(`7`), Item2: json.RawMessage(`{"min":7,"count":3}`)}+ assert.JSONEq(t, `{"min":7,"count":3}`, string(aggregatePartial("Min", both)))++ onlyItem := aggItem{Item: json.RawMessage(`7`)}+ assert.JSONEq(t, `{"min":7,"count":1}`, string(aggregatePartial("Min", onlyItem)))+ assert.JSONEq(t, `{"max":7,"count":1}`, string(aggregatePartial("Max", onlyItem)))++ assert.Nil(t, aggregatePartial("Min", aggItem{}))+ assert.Equal(t, json.RawMessage(`7`), aggregatePartial("Count", onlyItem))+}++// An aggregate over a partition that matched no row has no partial for the+// gateway to send, and the aggregate still has to finalize.+func TestFeedAggregatePartial_SkipsAnAbsentPartial(t *testing.T) {+ a := newAggregator("Min")+ require.NoError(t, feedAggregatePartial(a, "Min", aggItem{}))+ out, err := a.finalize()+ require.NoError(t, err)+ assert.Nil(t, out)++ require.NoError(t, feedAggregatePartial(a, "Min", aggItem{Item2: json.RawMessage(`{"count":0}`)}))+ out, err = a.finalize()+ require.NoError(t, err)+ assert.Nil(t, out)+}+// ---------------------------------------------------------- Min-func TestMinAgg_EmptyFinalizesToNull(t *testing.T) {+func TestMinAgg_EmptyFinalizesToUndefined(t *testing.T) {out := feedFinal(t, newAggregator("Min"))- assert.Equal(t, json.RawMessage(`null`), out)+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")}func TestMinAgg_SinglePartitionNumeric(t *testing.T) {@@ -128,9 +160,9 @@ func TestMinAgg_MixedTypesUsesCosmosOrdering(t *testing.T) {// ---------------------------------------------------------- Max-func TestMaxAgg_EmptyFinalizesToNull(t *testing.T) {+func TestMaxAgg_EmptyFinalizesToUndefined(t *testing.T) {out := feedFinal(t, newAggregator("Max"))- assert.Equal(t, json.RawMessage(`null`), out)+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")}func TestMaxAgg_AcrossPartitions_TakesLargest(t *testing.T) {diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.goindex 330eb408ae..397d34983e 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go@@ -57,6 +57,15 @@ type distinctPipeline struct {fanout rangeFanoutclosed bool+ // offset and limit are the plan's global skip and take, applied after+ // dedup: the gateway strips OFFSET/LIMIT and TOP from the rewritten query+ // so that each range reports its own distinct values in full and the page+ // is cut once, over the whole result. A negative limit means unlimited.+ offset int+ limit int+ skipped int+ taken int+// Pending items not yet returned to the caller. Populated during// ProvideData as each partition's response is de-duplicated against the// seen set.@@ -68,19 +77,30 @@ type distinctPipeline struct {seen map[string]struct{}}-// newDistinctPipeline builds a pipeline for a plan whose distinctType is-// "Unordered". Returns an error if the plan's distinctType is "Ordered" —-// that form composes with Stage 3's ORDER BY merge and is not handled here.-func newDistinctPipeline(plan *planDoc, pkRangeIDs []string) (*distinctPipeline, error) {- if plan.QueryInfo.DistinctType != "Unordered" {- // "Ordered" is Stage 3's responsibility; anything else is unexpected.+// newDistinctPipeline builds a pipeline for a DISTINCT plan. "Ordered" and+// "Unordered" differ only in whether the gateway promises sorted input, which+// a global seen-set does not need; any other distinctType is a shape the+// engine has not seen and rejects.+func newDistinctPipeline(plan *planDoc, pkRangeIDs []string, query string) (*distinctPipeline, error) {+ if plan.QueryInfo.DistinctType != "Unordered" && plan.QueryInfo.DistinctType != "Ordered" {return nil, ErrUnsupportedPlanFeature}- // DISTINCT plans carry an empty rewrittenQuery — run the original query- // against each partition. The pipeline needs the original query text to- // populate QueryRequest.Query; the caller passes it via CreateQueryPipeline.+ offset, limit, err := planSkipTake(plan.QueryInfo)+ if err != nil {+ return nil, err+ }+ // A plain DISTINCT plan carries no rewrittenQuery, so each range answers+ // the user's own query. A DISTINCT the user paged or capped does carry+ // one — the same DISTINCT with the OFFSET/LIMIT or TOP taken off — and+ // running that instead is what lets this pipeline cut the page globally.+ if plan.QueryInfo.RewrittenQuery != "" {+ query = plan.QueryInfo.RewrittenQuery+ }return &distinctPipeline{+ query: query,fanout: newRangeFanout(pkRangeIDs),+ offset: offset,+ limit: limit,seen: make(map[string]struct{}),}, nil}@@ -95,6 +115,11 @@ func (p *distinctPipeline) IsComplete() bool {if len(p.pending) > 0 {return false}+ if p.limit >= 0 && p.taken >= p.limit {+ // LIMIT satisfied — whatever the ranges still hold cannot change the+ // answer, so the pipeline stops asking for it.+ return true+ }return p.fanout.allDrained()}@@ -116,6 +141,9 @@ func (p *distinctPipeline) Run() (*queryengine.PipelineResult, error) {IsCompleted: p.IsComplete(),}, nil}+ if p.IsComplete() {+ return &queryengine.PipelineResult{IsCompleted: true}, nil+ }if reqs := p.fanout.requests(p.query); len(reqs) > 0 {return &queryengine.PipelineResult{Requests: reqs}, nil}@@ -124,7 +152,8 @@ func (p *distinctPipeline) Run() (*queryengine.PipelineResult, error) {// ProvideData implements queryengine.QueryPipeline. Each incoming partition// response is parsed and its documents are fed through the global seen set.-// First-sightings become pending items; duplicates are silently dropped.+// First-sightings pass to OFFSET/LIMIT and then become pending items;+// duplicates are silently dropped.func (p *distinctPipeline) ProvideData(results []queryengine.QueryResult) error {for _, r := range results {resp, err := parseDistinctPartitionResponse(r.Data)@@ -141,11 +170,19 @@ func (p *distinctPipeline) ProvideData(results []queryengine.QueryResult) errorcontinue}p.seen[key] = struct{}{}+ if p.skipped < p.offset {+ p.skipped+++ continue+ }+ if p.limit >= 0 && p.taken >= p.limit {+ break+ }// Preserve the partition's original document bytes in the output —// callers receive what the gateway returned, not our canonicalized// form. Copy defensively: json.RawMessage is aliased to the input// buffer, which the SDK may reuse for later pages.p.pending = append(p.pending, append([]byte(nil), doc...))+ p.taken++}if err := p.fanout.note(r); err != nil {return errdiff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.goindex 3e0be9e63f..fd55f7613f 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go@@ -88,12 +88,15 @@ func rawMsgs(ss ...string) []json.RawMessage {return out}+// distinctTestQuery stands in for the user's query, which a DISTINCT plan+// with no rewrittenQuery sends to every range unchanged.+const distinctTestQuery = `SELECT DISTINCT c.country FROM c`+func TestDistinctPipeline_SinglePartitionEmitsAll(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)- p.query = `SELECT DISTINCT c.country FROM c`items := feedSingle(t, p, rawMsgs(`{"country":"AD"}`, `{"country":"AE"}`, `{"country":"DE"}`))assert.Len(t, items, 3)@@ -103,7 +106,7 @@ func TestDistinctPipeline_SinglePartitionEmitsAll(t *testing.T) {func TestDistinctPipeline_DedupesWithinPartition(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)items := feedSingle(t, p, rawMsgs(`{"country":"AE"}`, `{"country":"AE"}`, `{"country":"AD"}`))@@ -113,7 +116,7 @@ func TestDistinctPipeline_DedupesWithinPartition(t *testing.T) {func TestDistinctPipeline_DedupesAcrossPartitions(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0", "1", "2"})+ p, err := newDistinctPipeline(plan, []string{"0", "1", "2"}, distinctTestQuery)require.NoError(t, err)// Kick off the request fan-out.@@ -144,7 +147,7 @@ func TestDistinctPipeline_DedupesAcrossPartitions(t *testing.T) {func TestDistinctPipeline_TreatsKeyReorderedObjectsAsDuplicates(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)items := feedSingle(t, p, rawMsgs(@@ -158,7 +161,7 @@ func TestDistinctPipeline_DistinctValueScalars(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"plan.QueryInfo.HasSelectValue = true- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)// DISTINCT VALUE emits bare scalars in Documents[].@@ -166,10 +169,10 @@ func TestDistinctPipeline_DistinctValueScalars(t *testing.T) {assert.Len(t, items, 3)}-func TestDistinctPipeline_RejectsOrderedDistinctType(t *testing.T) {+func TestDistinctPipeline_RejectsUnknownDistinctType(t *testing.T) {plan := &planDoc{}- plan.QueryInfo.DistinctType = "Ordered"- _, err := newDistinctPipeline(plan, []string{"0"})+ plan.QueryInfo.DistinctType = "SomethingNew"+ _, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.ErrorIs(t, err, ErrUnsupportedPlanFeature)}@@ -178,7 +181,7 @@ func TestDistinctPipeline_RejectsOrderedDistinctType(t *testing.T) {func TestDistinctPipeline_DedupesAcrossPages(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)rows, read := drivePages(t, p, map[string][]page{@@ -208,7 +211,7 @@ func mkDistinctBody(t *testing.T, values ...string) []byte {func TestDistinctPipeline_HighCardinality(t *testing.T) {plan := &planDoc{}plan.QueryInfo.DistinctType = "Unordered"- p, err := newDistinctPipeline(plan, []string{"0"})+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)require.NoError(t, err)_, _ = p.Run()@@ -221,3 +224,86 @@ func TestDistinctPipeline_HighCardinality(t *testing.T) {{PartitionKeyRangeID: "0", Data: body},}))}++// ------------------------------------------------- OFFSET, LIMIT and TOP++// A DISTINCT the user paged carries a rewrittenQuery with the OFFSET/LIMIT+// taken off, so every range reports its own distinct values in full and the+// page is cut once, here. Sending the user's query instead would make each+// range apply the LIMIT to itself.+func TestNewDistinctPipeline_PagedPlanRunsTheRewrittenQuery(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Limit = intPtr(3)+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`+ p, err := newDistinctPipeline(plan, []string{"0"}, `SELECT DISTINCT c.country FROM c OFFSET 0 LIMIT 3`)+ require.NoError(t, err)+ assert.Equal(t, `SELECT DISTINCT c.country FROM c`, p.Query())+}++func TestDistinctPipeline_LimitAppliesAfterDedup(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Limit = intPtr(2)+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)+ require.NoError(t, err)++ items := feedSingle(t, p, rawMsgs(`"AE"`, `"AE"`, `"BR"`, `"CN"`))+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(items))+}++func TestDistinctPipeline_OffsetSkipsDistinctValuesNotRows(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Offset = intPtr(1)+ plan.QueryInfo.Limit = intPtr(2)+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)+ require.NoError(t, err)++ items := feedSingle(t, p, rawMsgs(`"AE"`, `"AE"`, `"BR"`, `"CN"`, `"DE"`))+ assert.Equal(t, []string{`"BR"`, `"CN"`}, stringsOf(items))+}++// TOP on a DISTINCT reaches the engine as a plan field exactly like LIMIT,+// and caps the same way.+func TestDistinctPipeline_TopCapsTheDedupedStream(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Top = intPtr(2)+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)+ require.NoError(t, err)++ items := feedSingle(t, p, rawMsgs(`"AE"`, `"BR"`, `"CN"`))+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(items))+}++// Once the LIMIT is met the pipeline must stop, however many pages the ranges+// still hold: a satisfied page costs request charge for rows nobody sees.+func TestDistinctPipeline_StopsAtTheLimitWithoutReadingTheNextPage(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Limit = intPtr(2)+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)+ require.NoError(t, err)++ rows, read := drivePages(t, p, map[string][]page{+ "0": {+ {body: mkDistinctBody(t, `"AE"`, `"BR"`), continuation: "0-page-2"},+ {body: mkDistinctBody(t, `"CN"`, `"DE"`)},+ },+ })+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(rows))+ assert.Equal(t, map[string]int{"0": 1}, read)+}++func stringsOf(items [][]byte) []string {+ out := make([]string, 0, len(items))+ for _, it := range items {+ out = append(out, string(it))+ }+ return out+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go b/backend/plugin/db/cosmosdb/queryengine/gonative/engine.goindex 8dd9f06aa7..082715567d 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go@@ -1,38 +1,38 @@package gonativeimport (+ "reflect"+ "slices"+"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"+ "github.com/pkg/errors")-// Engine is the pure-Go implementation of queryengine.QueryEngine.-// It is constructed by Default(); callers should not instantiate Engine-// directly because future stages may add required initialization.-type Engine struct {- // (future stages add fields: supportedFeatures set, cardinality caps, etc.)-}+// Engine is the pure-Go implementation of queryengine.QueryEngine. Construct+// it with Default() and treat the value as opaque.+type Engine struct{}// Compile-time check that *Engine satisfies the interface.var _ queryengine.QueryEngine = (*Engine)(nil)-// Default returns a new Engine with Stage-0 capabilities (none).-// Callers should treat the returned value as opaque.+// Default returns a new Engine.func Default() *Engine {return &Engine{}}// supportedFeatures enumerates the gateway-known feature names the engine// can serve. The string is comma-separated per the gateway's query-plan-// protocol; new feature names are appended as each stage lands.+// protocol.//-// Aggregate — VALUE-form aggregate queries (Stage 1)-// NonValueAggregate — aliased-form aggregates (Stage 1)-// MultipleAggregates — multi-aggregate in a single SELECT (Stage 1)-// DistinctValue — SELECT DISTINCT VALUE c.field FROM c (Stage 2)-// Distinct — SELECT DISTINCT c.field FROM c — object form (Stage 2)-// OrderBy — single-key ORDER BY (Stage 3)-// Top — SELECT TOP N FROM c (Stage 4)-// OffsetAndLimit — SELECT … ORDER BY … OFFSET N LIMIT M (Stage 5)-// GroupBy — SELECT … FROM c GROUP BY c.field (Stage 6)+// Aggregate — VALUE-form aggregate queries+// NonValueAggregate — aliased-form aggregates+// MultipleAggregates — multi-aggregate in a single SELECT+// DistinctValue — SELECT DISTINCT VALUE c.field FROM c+// Distinct — SELECT DISTINCT c.field FROM c — object form+// OrderBy — single-key ORDER BY+// Top — SELECT TOP N FROM c+// OffsetAndLimit — OFFSET N LIMIT M+// GroupBy — SELECT … FROM c GROUP BY c.fieldconst supportedFeatures = "Aggregate,NonValueAggregate,MultipleAggregates,Distinct,DistinctValue,OrderBy,Top,OffsetAndLimit,GroupBy"// SupportedFeatures implements queryengine.QueryEngine.@@ -40,85 +40,117 @@ func (*Engine) SupportedFeatures() string {return supportedFeatures}+// Names of the planQueryInfo fields a dispatch branch claims. A plan field the+// chosen pipeline does not claim fails the query — see requireHandled.+const (+ fieldRewrittenQuery = "RewrittenQuery"+ fieldHasSelectValue = "HasSelectValue"+ fieldAggregates = "Aggregates"+ fieldGroupByAliases = "GroupByAliases"+ fieldGroupByAliasToAggregateType = "GroupByAliasToAggregateType"+ fieldDistinctType = "DistinctType"+ fieldTop = "Top"+ fieldOffset = "Offset"+ fieldLimit = "Limit"+ fieldOrderBy = "OrderBy"+ fieldOrderByExpressions = "OrderByExpressions"+ fieldGroupByExpressions = "GroupByExpressions"+)+// CreateQueryPipeline implements queryengine.QueryEngine. It parses the plan,-// rejects any feature the engine does not yet support, then dispatches to the-// appropriate pipeline implementation.+// picks the pipeline for the plan's shape and rejects any plan field that+// pipeline cannot honour.func (*Engine) CreateQueryPipeline(query string, plan string, pkranges string) (queryengine.QueryPipeline, error) {p, err := parsePlan([]byte(plan))if err != nil {return nil, err}- if err := rejectUnsupportedPlan(p); err != nil {- return nil, err- }rangeIDs, err := parsePKRangeIDs([]byte(pkranges))if err != nil {return nil, err}+ qi := p.QueryInfoswitch {- case len(p.QueryInfo.OrderBy) > 0:+ case len(qi.OrderBy) > 0:+ // ORDER BY owns every operator that applies to an ordered stream:+ // DISTINCT dedups it, TOP and OFFSET/LIMIT cut it.+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldOrderBy, fieldOrderByExpressions,+ fieldDistinctType, fieldTop, fieldOffset, fieldLimit); err != nil {+ return nil, err+ }return newOrderByPipeline(p, rangeIDs)- case p.QueryInfo.DistinctType == "Unordered":- pipe, err := newDistinctPipeline(p, rangeIDs)- if err != nil {+ case qi.DistinctType != "":+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldDistinctType, fieldTop, fieldOffset, fieldLimit); err != nil {return nil, err}- pipe.query = query- return pipe, nil- case len(p.QueryInfo.Aggregates) > 0:- pipe, err := newValuePipeline(p, rangeIDs)- if err != nil {+ return newDistinctPipeline(p, rangeIDs, query)+ case len(qi.Aggregates) > 0:+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldAggregates, fieldOffset, fieldLimit); err != nil {return nil, err}- return pipe, nil- case len(p.QueryInfo.GroupByExpressions) > 0:+ return newValuePipeline(p, rangeIDs)+ case len(qi.GroupByExpressions) > 0:// GROUP BY plans carry groupByExpressions; the aliased-aggregate- // (single-group) path used by Stage 1 has groupByAliasToAggregateType- // populated but groupByExpressions empty. The check order matters.+ // (single-group) path has groupByAliasToAggregateType populated but+ // groupByExpressions empty. The check order matters.+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldGroupByExpressions, fieldGroupByAliases, fieldGroupByAliasToAggregateType); err != nil {+ return nil, err+ }return newGroupPipeline(p, rangeIDs)- case len(p.QueryInfo.GroupByAliasToAggregateType) > 0:- pipe, err := newAliasedPipeline(p, rangeIDs)- if err != nil {+ case len(qi.GroupByAliasToAggregateType) > 0:+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldGroupByAliases, fieldGroupByAliasToAggregateType, fieldOffset, fieldLimit); err != nil {return nil, err}- return pipe, nil- case p.QueryInfo.Top != nil:- pipe, err := newTopPipeline(p, rangeIDs)- if err != nil {+ return newAliasedPipeline(p, rangeIDs)+ case qi.Top != nil || qi.Offset != nil || qi.Limit != nil:+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,+ fieldTop, fieldOffset, fieldLimit); err != nil {return nil, err}- pipe.query = query- return pipe, nil+ return newSkipTakePipeline(p, rangeIDs, query)default:- return nil, ErrUnsupportedPlanFeature+ return nil, errors.Wrap(ErrUnsupportedPlanFeature, "no pipeline for this plan shape")}}-// rejectUnsupportedPlan returns ErrUnsupportedPlanFeature when the plan uses-// any feature beyond the engine's current scope. Later stages will prune-// entries from this guard as their operators come online.-func rejectUnsupportedPlan(p *planDoc) error {- qi := p.QueryInfo- // "Ordered" DISTINCT composes with ORDER BY merge; reject until Stage 3- // wires composition through the merge heap. Stage 3 itself handles plain- // ORDER BY — see the dispatch above.- if qi.DistinctType == "Ordered" {- return ErrUnsupportedPlanFeature- }- if qi.DistinctType != "" && qi.DistinctType != "None" && qi.DistinctType != "Unordered" {- return ErrUnsupportedPlanFeature- }- // OFFSET/LIMIT always composes with ORDER BY per Cosmos syntax; when that- // composition lands the plan has both qi.OrderBy and qi.Offset/qi.Limit,- // and the dispatch routes to newOrderByPipeline which handles them.- // Plans that set offset/limit without an ORDER BY fall through to the- // default case in CreateQueryPipeline and are rejected there.- if qi.HasNonStreamingOrderBy {- return ErrUnsupportedPlanFeature+// requireHandled rejects a plan whose queryInfo sets a field the chosen+// pipeline does not claim. It walks planQueryInfo by reflection rather than+// checking a fixed list, so a field added to the struct later is rejected+// until a pipeline claims it: an operator the pipeline cannot apply must fail+// the query rather than vanish from the answer.+func requireHandled(qi planQueryInfo, handled ...string) error {+ v := reflect.ValueOf(qi)+ t := v.Type()+ for i := range t.NumField() {+ name := t.Field(i).Name+ if !planFieldSet(v.Field(i)) || slices.Contains(handled, name) {+ continue+ }+ return errors.Wrapf(ErrUnsupportedPlanFeature, "pipeline cannot honour queryInfo.%s", name)}return nil}+// planFieldSet reports whether a queryInfo field carries an instruction. A+// plan that spells a field out as empty — `"aggregates": []` — says nothing,+// and reflect.Value.IsZero would call that slice set because it is non-nil.+// A pointer field is set whenever it is non-nil: `"top": 0` means TOP 0.+func planFieldSet(v reflect.Value) bool {+ switch v.Kind() {+ case reflect.Slice, reflect.Map, reflect.String:+ return v.Len() > 0+ case reflect.Pointer:+ return !v.IsNil()+ default:+ return !v.IsZero()+ }+}+// CreateReadManyPipeline implements queryengine.QueryEngine.func (*Engine) CreateReadManyPipeline(_ []queryengine.ItemIdentity, _ string, _ string, _ uint8, _ []string) (queryengine.QueryPipeline, error) {return nil, ErrUnsupportedPlanFeaturediff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.goindex 8bf2d215f9..08eb6f5694 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go@@ -1,31 +1,28 @@-package gonative_test+package gonativeimport (- "errors"+ "os"+ "path/filepath""testing""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/plugin/db/cosmosdb/queryengine/gonative")func TestDefaultReturnsNonNilEngine(t *testing.T) {- e := gonative.Default()+ e := Default()require.NotNil(t, e)}-func TestDefaultAdvertisesStage1Features(t *testing.T) {- // Stage 1 advertises aggregate-family features so the gateway emits plans- // for VALUE + aliased + multi-aggregate queries.- feats := gonative.Default().SupportedFeatures()+func TestDefaultAdvertisesAggregateFeatures(t *testing.T) {+ feats := Default().SupportedFeatures()assert.Contains(t, feats, "Aggregate")assert.Contains(t, feats, "NonValueAggregate")assert.Contains(t, feats, "MultipleAggregates")}func TestDefaultCreateQueryPipelineRejectsUnparseablePlan(t *testing.T) {- _, err := gonative.Default().CreateQueryPipeline("SELECT * FROM c", "not json", "{}")+ _, err := Default().CreateQueryPipeline("SELECT * FROM c", "not json", "{}")require.Error(t, err)}@@ -34,20 +31,115 @@ func TestDefaultCreateQueryPipelineRejectsUnsupportedPlan(t *testing.T) {// is out of the engine's scope — the design spec explicitly excludes it.plan := `{"partitionedQueryExecutionInfoVersion":2,"queryInfo":{"hasNonStreamingOrderBy":true}}`pkranges := `{"PartitionKeyRanges":[{"id":"0"}]}`- _, err := gonative.Default().CreateQueryPipeline("SELECT ...", plan, pkranges)- require.Error(t, err)- assert.True(t, errors.Is(err, gonative.ErrUnsupportedPlanFeature))+ _, err := Default().CreateQueryPipeline("SELECT ...", plan, pkranges)+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)}func TestDefaultCreateReadManyPipelineReturnsUnsupported(t *testing.T) {- _, err := gonative.Default().CreateReadManyPipeline(nil, "{}", "", 0, nil)- require.Error(t, err)- assert.True(t, errors.Is(err, gonative.ErrUnsupportedPlanFeature))+ _, err := Default().CreateReadManyPipeline(nil, "{}", "", 0, nil)+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)}func TestDefaultReturnsDistinctEngineInstances(t *testing.T) {// Each caller gets its own instance — enables per-engine caches in later stages.- a := gonative.Default()- b := gonative.Default()+ a := Default()+ b := Default()assert.NotSame(t, a, b)}++// recordedPlan loads a query plan the gateway really emitted for one of the+// BYT-10397 query shapes, as the engine receives it.+func recordedPlan(t *testing.T, name string) string {+ t.Helper()+ raw, err := os.ReadFile(filepath.Join("testdata", "query_"+name+"_plan.json"))+ require.NoError(t, err)+ return string(raw)+}++// TestCreateQueryPipelineRejectsUnhandledPlanField drives the dispatch with+// plans whose composition no pipeline implements. The query has to fail rather+// than come back missing an operator, which is what every wrong row count in+// BYT-10397 was.+func TestCreateQueryPipelineRejectsUnhandledPlanField(t *testing.T) {+ const pkranges = `{"PartitionKeyRanges":[{"id":"0"},{"id":"1"}]}`+ for _, tc := range []struct {+ plan string+ query string+ field string+ }{+ {"groupby_top", `SELECT TOP 3 c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`, "Top"},+ {"groupby_offset_limit", `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket OFFSET 1 LIMIT 2`, "Offset"},+ } {+ t.Run(tc.plan, func(t *testing.T) {+ _, err := Default().CreateQueryPipeline(tc.query, recordedPlan(t, tc.plan), pkranges)+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)+ assert.Contains(t, err.Error(), "queryInfo."+tc.field)+ })+ }+}++// TestCreateQueryPipelineAcceptsRecordedCompositions pins the dispatch against+// the plan shapes BYT-10397 collected: each composes two operators, and the+// pipeline the dispatch picks has to claim both of them.+func TestCreateQueryPipelineAcceptsRecordedCompositions(t *testing.T) {+ const pkranges = `{"PartitionKeyRanges":[{"id":"0"},{"id":"1"}]}`+ for _, tc := range []struct{ plan, query string }{+ {"distinct_orderby", `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket`},+ {"distinct_orderby_value", `SELECT DISTINCT VALUE c.bucket FROM c ORDER BY c.bucket`},+ {"top_orderby_asc", `SELECT TOP 5 c.seq FROM c ORDER BY c.seq ASC`},+ {"top_orderby_desc", `SELECT TOP 5 c.seq FROM c ORDER BY c.seq DESC`},+ {"distinct_value_limit", `SELECT DISTINCT VALUE c.bucket FROM c OFFSET 0 LIMIT 3`},+ {"distinct_offset_limit", `SELECT DISTINCT c.bucket FROM c OFFSET 2 LIMIT 3`},+ {"distinct_top", `SELECT DISTINCT TOP 3 c.bucket FROM c`},+ {"offset_limit", `SELECT c.seq FROM c OFFSET 2 LIMIT 3`},+ {"agg_offset_limit", `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`},+ {"min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 99999999`},+ {"min_aliased_empty", `SELECT MIN(c.seq) AS m FROM c WHERE c.seq > 99999999`},+ {"groupby_plain", `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`},+ } {+ t.Run(tc.plan, func(t *testing.T) {+ pipe, err := Default().CreateQueryPipeline(tc.query, recordedPlan(t, tc.plan), pkranges)+ require.NoError(t, err)+ pipe.Close()+ })+ }+}++// A GROUP BY that the user also ordered reaches the engine as a plan with no+// order fields at all — the ordering survives only inside rewrittenQuery — so+// the guard cannot see it. The group pipeline reads the direction off the+// gateway's own per-partition streams instead.+func TestCreateQueryPipelineAcceptsGroupByWithHiddenOrderBy(t *testing.T) {+ raw := recordedPlan(t, "groupby_orderby")+ plan, err := parsePlan([]byte(raw))+ require.NoError(t, err)+ assert.Empty(t, plan.QueryInfo.OrderBy)+ assert.Empty(t, plan.QueryInfo.OrderByExpressions)+ assert.Contains(t, plan.QueryInfo.RewrittenQuery, "ORDER BY")++ pipe, err := Default().CreateQueryPipeline(+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket`,+ raw, `{"PartitionKeyRanges":[{"id":"0"}]}`)+ require.NoError(t, err)+ pipe.Close()+}++func TestRequireHandledWalksEveryPlanField(t *testing.T) {+ n := 0+ qi := planQueryInfo{RewrittenQuery: "SELECT 1", Top: &n}++ require.NoError(t, requireHandled(qi, fieldRewrittenQuery, fieldTop))++ err := requireHandled(qi, fieldRewrittenQuery)+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)+ assert.Contains(t, err.Error(), "queryInfo.Top", "TOP 0 is an instruction, not an absent field")+}++// A plan that spells a field out as empty asks for nothing, so it must not+// make the dispatch reject a shape it can serve.+func TestRequireHandledIgnoresFieldsSpeltOutEmpty(t *testing.T) {+ p, err := parsePlan([]byte(`{"queryInfo":{"aggregates":[],"groupByAliases":[],` ++ `"groupByAliasToAggregateType":{},"distinctType":"None","orderBy":[]}}`))+ require.NoError(t, err)+ require.NoError(t, requireHandled(p.QueryInfo))+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group.goindex 42b070155e..e13f1ea13b 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go@@ -2,16 +2,16 @@ package gonativeimport ("encoding/json"- "fmt"+ "slices""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine""github.com/pkg/errors")// groupByItem is one entry in the per-document groupByItems array. For a-// single-key GROUP BY the array has one element; multi-key GROUP BY (not in-// Stage 6's target inventory but straightforward to support since we hash the-// whole array) would have more.+// single-key GROUP BY the array has one element; multi-key GROUP BY would+// have more, and needs no special handling here because the engine hashes and+// orders the whole array.type groupByItem struct {Item json.RawMessage `json:"item"`}@@ -44,6 +44,9 @@ type groupState struct {// report a given group populates these; subsequent partitions overwrite// with the same value (Cosmos guarantees equality).keyValues map[string]json.RawMessage+ // items is the group's key as the gateway reported it, kept for ordering+ // the output rows.+ items []groupByItem// aggregators are ordered per groupByAliases; entries whose alias is a// group-key column are nil.aggregators []aggregator@@ -71,6 +74,20 @@ type groupPipeline struct {// groups is the per-group state, keyed on the canonical JSON of// groupByItems. Grows to the cardinality of distinct group keys.groups map[string]*groupState+ // order holds those keys in the order the groups were first seen, which+ // is what the output falls back to when the keys cannot be ordered.+ order []string++ // descending says which way to order the output rows, and+ // directionKnown whether the data has told us yet. A GROUP BY the user+ // also ordered+ // reaches the engine as a plan with no order fields at all — the ORDER BY+ // survives only inside rewrittenQuery — so the direction can be read only+ // off the responses: the gateway returns each partition's own groups in+ // the requested order, and the first adjacent pair that differs says+ // which way they run.+ descending bool+ directionKnown bool// pending holds fully finalized rows awaiting the caller's next Run().pending [][]byte@@ -84,10 +101,9 @@ type groupPipeline struct {// and groupByAliasToAggregateType fields; any alias missing from the map is// treated as a group-key column.//-// Returns ErrUnsupportedPlanFeature when the plan composes GROUP BY with-// operators this stage does not handle (ORDER BY, DISTINCT, TOP, OFFSET/LIMIT).-// Those compositions are valid SQL but require wrapping groupPipeline in-// additional operators — deferred.+// Composing GROUP BY with TOP, OFFSET/LIMIT or DISTINCT would mean wrapping+// this pipeline in another operator; CreateQueryPipeline rejects those plans+// before they reach here.func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error) {qi := plan.QueryInfoif len(qi.GroupByExpressions) == 0 {@@ -96,13 +112,6 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, errorif len(qi.GroupByAliases) == 0 {return nil, errors.New("gonative: groupPipeline requires groupByAliases")}- // Stage 6 does not compose GROUP BY with other operators. Reject anything- // that would require wrapping in ORDER BY / TOP / OFFSET-LIMIT / DISTINCT.- if len(qi.OrderBy) > 0 || qi.Top != nil || qi.Offset != nil || qi.Limit != nil ||- qi.DistinctType == "Unordered" || qi.DistinctType == "Ordered" {- return nil, ErrUnsupportedPlanFeature- }-aliases := append([]string(nil), qi.GroupByAliases...)kinds := make([]string, len(aliases))for i, alias := range aliases {@@ -184,6 +193,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {if err != nil {return err}+ p.noteOrdering(resp.Documents)for docIdx, doc := range resp.Documents {if len(doc.GroupByItems) == 0 {return errors.Errorf("gonative: group doc %d missing groupByItems", docIdx)@@ -196,6 +206,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {if !ok {state = p.initGroupState(doc)p.groups[key] = state+ p.order = append(p.order, key)}// Merge this partition's aggregate contributions into the group's state.for i, alias := range p.aliases {@@ -229,6 +240,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {s := &groupState{keyValues: make(map[string]json.RawMessage, len(p.aliases)),+ items: append([]groupByItem(nil), doc.GroupByItems...),aggregators: make([]aggregator, len(p.aliases)),}for i, alias := range p.aliases {@@ -244,13 +256,13 @@ func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {return s}-// finalize walks the group map and produces one output row per group.-// Emission order is unspecified (follows Go's map iteration randomization);-// callers that need deterministic order should add an ORDER BY clause, which-// a future composition stage will wrap around this pipeline.+// finalize produces one output row per group, in group-key order — see+// sortGroups. Walking p.groups instead would emit them in Go's randomized map+// order, which made the same query answer in a different order every run.func (p *groupPipeline) finalize() error {- for _, state := range p.groups {- row, err := p.buildRow(state)+ p.sortGroups()+ for _, key := range p.order {+ row, err := p.buildRow(p.groups[key])if err != nil {return err}@@ -260,36 +272,106 @@ func (p *groupPipeline) finalize() error {return nil}+// noteOrdering reads the output direction off one partition's response, which+// is the only place a GROUP BY's ORDER BY reaches the engine. Gives up on a+// key it cannot compare: sortGroups checks for those itself.+func (p *groupPipeline) noteOrdering(docs []groupDoc) {+ if p.directionKnown {+ return+ }+ for i := 1; i < len(docs); i++ {+ order, err := compareGroupKeys(docs[i-1].GroupByItems, docs[i].GroupByItems)+ if err != nil {+ return+ }+ if order != 0 {+ p.descending = order > 0+ p.directionKnown = true+ return+ }+ }+}++// sortGroups puts p.order in group-key order, the direction taken from the+// gateway's own per-partition streams. A GROUP BY without an ORDER BY has no+// defined output order, so ordering those rows too costs nothing and is what+// makes every run answer the same way.+//+// Leaves first-seen order in place when any group key is a shape Cosmos+// cannot order (an array or an object), since the comparator would have to+// invent an answer for those.+func (p *groupPipeline) sortGroups() {+ for _, key := range p.order {+ for _, item := range p.groups[key].items {+ if detectJSONType(item.Item) == jsonUnorderable {+ return+ }+ }+ }+ slices.SortStableFunc(p.order, func(a, b string) int {+ order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items)+ if err != nil {+ // Unreachable: every key was checked orderable just above.+ return 0+ }+ if p.descending {+ return -order+ }+ return order+ })+}++// compareGroupKeys orders two group keys under Cosmos item ordering, column+// by column.+func compareGroupKeys(a, b []groupByItem) (int, error) {+ for i := range min(len(a), len(b)) {+ order, err := compareItems(a[i].Item, b[i].Item)+ if err != nil {+ return 0, err+ }+ if order != 0 {+ return order, nil+ }+ }+ return len(a) - len(b), nil+}+// buildRow assembles the final JSON object for a single group, preserving// the alias order from groupByAliases. JSON object key order in Go's encoder// depends on sort for map marshaling, so we write the object manually.+//+// An alias whose aggregate has no value — MIN or MAX over a group where the+// column is undefined in every row — is left out of the object, which is how+// the gateway spells it.func (p *groupPipeline) buildRow(s *groupState) ([]byte, error) {var buf []bytebuf = append(buf, '{')for i, alias := range p.aliases {- if i > 0 {- buf = append(buf, ',')- }- aliasJSON, err := json.Marshal(alias)- if err != nil {- return nil, err- }- buf = append(buf, aliasJSON...)- buf = append(buf, ':')+ value := json.RawMessage("null")if p.kinds[i] == "" {- v, ok := s.keyValues[alias]- if !ok {- buf = append(buf, []byte("null")...)- continue+ if v, ok := s.keyValues[alias]; ok {+ value = v}- buf = append(buf, v...)} else {final, err := s.aggregators[i].finalize()if err != nil {return nil, err}- buf = append(buf, final...)+ if final == nil {+ continue+ }+ value = final+ }+ if len(buf) > 1 {+ buf = append(buf, ',')+ }+ aliasJSON, err := json.Marshal(alias)+ if err != nil {+ return nil, err}+ buf = append(buf, aliasJSON...)+ buf = append(buf, ':')+ buf = append(buf, value...)}buf = append(buf, '}')return buf, nil@@ -323,26 +405,12 @@ func groupKey(items []groupByItem) (string, error) {// "count":...}]}. For Count/Sum/Avg the aggregator consumes "item" directly.// For Min/Max: prefer "item2" (the reliable cross-partition partial carrying// count); fall back to synthesizing a single-count partial from "item" when-// item2 is absent. Mirrors the feedAliasedPartial logic from Stage 1's-// aliased-aggregate pipeline.+// item2 is absent — see aggregatePartial, which both aggregate pipelines and+// this one route through.func extractAggregatePartial(kind string, raw json.RawMessage) (json.RawMessage, error) {- var wrap aliasedItem+ var wrap aggItemif err := json.Unmarshal(raw, &wrap); err != nil {return nil, errors.Wrap(err, "gonative: extract aggregate partial")}- switch kind {- case "Min":- if len(wrap.Item2) > 0 {- return wrap.Item2, nil- }- return fmt.Appendf(nil, `{"min":%s,"count":1}`, wrap.Item), nil- case "Max":- if len(wrap.Item2) > 0 {- return wrap.Item2, nil- }- return fmt.Appendf(nil, `{"max":%s,"count":1}`, wrap.Item), nil- default:- // Count, Sum, Avg — Item alone carries the partial.- return wrap.Item, nil- }+ return aggregatePartial(kind, wrap), nil}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.goindex e60dd2614c..bd4ffd12c9 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go@@ -103,28 +103,6 @@ func TestNewGroupPipeline_RequiresGroupByAliases(t *testing.T) {assert.Contains(t, err.Error(), "groupByAliases")}-func TestNewGroupPipeline_RejectsCompositionWithOrderBy(t *testing.T) {- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})- plan.QueryInfo.OrderBy = []string{"Ascending"}- _, err := newGroupPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)-}--func TestNewGroupPipeline_RejectsCompositionWithTop(t *testing.T) {- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})- n := 5- plan.QueryInfo.Top = &n- _, err := newGroupPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)-}--func TestNewGroupPipeline_RejectsCompositionWithDistinct(t *testing.T) {- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})- plan.QueryInfo.DistinctType = "Unordered"- _, err := newGroupPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)-}-// ---------------------------------------------------------- semanticsfunc TestGroupPipeline_SinglePartition_GroupKeyPassthrough(t *testing.T) {@@ -352,3 +330,123 @@ func TestExtractAggregatePartial_MaxSynthesizesFromItemWhenItem2Absent(t *testinrequire.NoError(t, err)assert.JSONEq(t, `{"max":99,"count":1}`, string(got))}++// ---------------------------------------------------------- output order++// groupKeysOf returns the group-key column of every emitted row, in the order+// the pipeline emitted them.+func groupKeysOf(t *testing.T, rows [][]byte, keyAlias string) []string {+ t.Helper()+ out := make([]string, 0, len(rows))+ for _, r := range rows {+ var obj map[string]any+ require.NoError(t, json.Unmarshal(r, &obj))+ v, ok := obj[keyAlias].(string)+ require.True(t, ok, "row missing %s key: %s", keyAlias, r)+ out = append(out, v)+ }+ return out+}++func countByCountryPlan() *planDoc {+ return newGroupPlan("c.country",+ []string{"country", "n"},+ map[string]string{"country": "", "n": "Count"})+}++// A GROUP BY the user also ordered reaches the engine as a plan with no order+// fields, so the direction comes off the data: the gateway returns each+// partition's own groups in the requested order.+func TestGroupPipeline_EmitsAscendingWhenThePartitionsRunAscending(t *testing.T) {+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t,+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},+ ),+ "1": mkGroupBody(t,+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},+ ),+ })+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))+}++func TestGroupPipeline_EmitsDescendingWhenThePartitionsRunDescending(t *testing.T) {+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t,+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},+ ),+ "1": mkGroupBody(t,+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},+ ),+ })+ assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))+}++// Without an ORDER BY the output order is unspecified, but it still has to be+// the same on every run: before this it followed Go's map iteration and five+// consecutive runs of one query gave five different orders.+func TestGroupPipeline_OutputOrderIsStableAcrossRuns(t *testing.T) {+ feed := map[string][]byte{+ "0": mkGroupBody(t,+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},+ groupEntry{key: `"US"`, payload: `{"country":"US","n":{"item":9}}`},+ ),+ }+ var first []string+ for range 5 {+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0"})+ require.NoError(t, err)+ got := groupKeysOf(t, driveGroup(t, p, feed), "country")+ if first == nil {+ first = got+ continue+ }+ assert.Equal(t, first, got)+ }+}++// A group key Cosmos cannot order — an array or an object — leaves the output+// in first-seen order rather than asking the comparator to invent one.+func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {+ plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})+ p, err := newGroupPipeline(plan, []string{"0"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t,+ groupEntry{key: `["b"]`, payload: `{"tags":["b"],"n":{"item":1}}`},+ groupEntry{key: `["a"]`, payload: `{"tags":["a"],"n":{"item":2}}`},+ ),+ })+ require.Len(t, rows, 2)+ assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[0]))+ assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[1]))+}++// MIN over a group where the column is undefined in every row has no value,+// and the gateway leaves that alias out of the row rather than writing null.+func TestGroupPipeline_OmitsAnAliasWithNoValue(t *testing.T) {+ plan := newGroupPlan("c.country", []string{"country", "lo"}, map[string]string{"country": "", "lo": "Min"})+ p, err := newGroupPipeline(plan, []string{"0"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t,+ groupEntry{key: `"AE"`, payload: `{"country":"AE","lo":{"item2":{"count":0}}}`},+ ),+ })+ require.Len(t, rows, 1)+ assert.JSONEq(t, `{"country":"AE"}`, string(rows[0]))+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.goindex 1ac99cd4cf..5203d9af7e 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go@@ -19,8 +19,8 @@ import (const orderByQueryFilterPlaceholder = "{documentdb-formattableorderbyquery-filter}"// orderByItem is each element of the per-document orderByItems array. The-// gateway emits one per ORDER BY expression; Stage 3 only supports single-key-// ORDER BY so we read orderByItems[0].+// gateway emits one per ORDER BY expression; the engine supports single-key+// ORDER BY only, so it reads orderByItems[0].type orderByItem struct {Item json.RawMessage `json:"item"`}@@ -113,17 +113,22 @@ func (h *mergeHeap) Pop() any {// ORDER BY expression. It emits rows in the requested order by k-way-merging// each partition's gateway-sorted stream.//-// Stage 3 scope is single-key ORDER BY only. Multi-key ORDER BY requires a-// Cosmos composite index on the container — if the container lacks one the-// gateway rejects the query outright; if it has one the gateway emits a plan-// with multiple entries in orderByExpressions, and the comparator would need-// to walk all of them. Both cases are deferred.+// Single-key ORDER BY only. Multi-key ORDER BY requires a Cosmos composite+// index on the container — if the container lacks one the gateway rejects the+// query outright; if it has one the gateway emits a plan with multiple entries+// in orderByExpressions, and the comparator would need to walk all of them.+// Both cases are deferred.//-// Stage 5 extends the pipeline with OFFSET/LIMIT awareness. Per Cosmos-// semantics, OFFSET/LIMIT always composes with ORDER BY; the gateway rewrites-// the per-partition query to `OFFSET 0 LIMIT (offset+limit)` so every-// partition delivers enough rows to satisfy any interleaving, and this-// pipeline handles the global skip + take after merging.+// The operators ORDER BY composes with all apply to the merged stream, after+// the k-way merge has put the rows in order:+//+// OFFSET/LIMIT — the gateway widens the per-partition query to+// `OFFSET 0 LIMIT (offset+limit)` so every partition delivers enough rows+// to cover any interleaving, and the global skip + take happens here.+// TOP — the per-partition query carries `TOP n`, so each partition delivers+// its own first n and the global first n is among them.+// DISTINCT — the payload of each merged row is deduplicated against every+// row already emitted, before OFFSET/LIMIT counts it.//// A range answers over as many pages as its share of the result needs. The// merge therefore runs only while every range that still has pages has a row@@ -139,22 +144,28 @@ type orderByPipeline struct {closed bool- // offset and limit implement plan.QueryInfo.Offset/Limit. A negative limit- // (the sentinel returned by newOrderByPipeline when the plan has no limit)- // means "unlimited". skipped counts rows dropped by offset; emitted counts- // rows forwarded to the caller — both drive the skip+take logic in drain.+ // offset and limit are the plan's global skip and take; a negative limit+ // means unlimited. skipped counts rows dropped by offset; emitted counts+ // rows forwarded to the caller — both drive the skip+take logic in merge.offset intlimit intskipped intemitted int+ // seen is the DISTINCT seen-set, nil unless the plan carries a DISTINCT.+ // Keys are the canonical JSON of an emitted payload, as in+ // distinctPipeline: a sorted stream makes duplicates adjacent, but the+ // service reports distinctType "Unordered" for the object form even under+ // ORDER BY, so the engine keeps the whole set rather than the last row.+ seen map[string]struct{}+// pending emits produced by k-way merging when all partitions have data.pending [][]byte}// newOrderByPipeline constructs a pipeline for a plan with exactly one ORDER BY// expression. Returns ErrUnsupportedPlanFeature when the plan names multiple-// ORDER BY keys — Stage 3 doesn't yet support that composition.+// ORDER BY keys, which the comparator cannot walk.func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, error) {if len(plan.QueryInfo.OrderBy) != 1 {return nil, ErrUnsupportedPlanFeature@@ -167,21 +178,14 @@ func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, e// the whole range and relies on the continuation token to resume it.query := strings.ReplaceAll(plan.QueryInfo.RewrittenQuery, orderByQueryFilterPlaceholder, "true")- // OFFSET/LIMIT always composes with ORDER BY per Cosmos syntax; defaults- // mean "no skip, unlimited take".- offset := 0- if plan.QueryInfo.Offset != nil {- if *plan.QueryInfo.Offset < 0 {- return nil, errors.Errorf("gonative: negative OFFSET=%d", *plan.QueryInfo.Offset)- }- offset = *plan.QueryInfo.Offset+ offset, limit, err := planSkipTake(plan.QueryInfo)+ if err != nil {+ return nil, err}- limit := -1- if plan.QueryInfo.Limit != nil {- if *plan.QueryInfo.Limit < 0 {- return nil, errors.Errorf("gonative: negative LIMIT=%d", *plan.QueryInfo.Limit)- }- limit = *plan.QueryInfo.Limit++ var seen map[string]struct{}+ if plan.QueryInfo.DistinctType != "" {+ seen = make(map[string]struct{})}partByID := make(map[string]int, len(pkRangeIDs))@@ -198,6 +202,7 @@ func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, epartitions: partitions,offset: offset,limit: limit,+ seen: seen,}, nil}@@ -235,7 +240,9 @@ func (p *orderByPipeline) IsComplete() bool {// 2. Otherwise ask for the next page of every range the merge is waiting on// — the initial fan-out across all ranges on the first call.func (p *orderByPipeline) Run() (*queryengine.PipelineResult, error) {- p.merge()+ if err := p.merge(); err != nil {+ return nil, err+ }if len(p.pending) > 0 {batch := p.pending@@ -306,13 +313,14 @@ func (p *orderByPipeline) ProvideData(results []queryengine.QueryResult) error {// still has pages ends the turn instead, because its next page may sort ahead// of every other range's head.//-// OFFSET/LIMIT (Stage 5) apply here: the first `offset` rows are discarded,-// and emission stops once `limit` rows have been collected (when limit >= 0).-// Both are tracked on the pipeline so the next turn resumes where this one-// stopped.-func (p *orderByPipeline) merge() {+// DISTINCT, OFFSET and LIMIT apply here, in that order: a row that repeats an+// already-emitted payload is dropped before it counts against OFFSET, the+// first `offset` surviving rows are discarded, and emission stops once `limit`+// rows have been collected (when limit >= 0). All three are tracked on the+// pipeline so the next turn resumes where this one stopped.+func (p *orderByPipeline) merge() error {if p.waitingForPage() {- return+ return nil}h := &mergeHeap{partitions: p.partitions, descending: p.descending}for i, s := range p.partitions {@@ -325,18 +333,25 @@ func (p *orderByPipeline) merge() {for h.Len() > 0 {if p.limit >= 0 && p.emitted >= p.limit {// LIMIT reached — the caller has everything it asked for.- return+ return nil}top, ok := heap.Pop(h).(int)if !ok {// Unreachable: the heap is seeded with partition indexes only.- return+ return nil}s := p.partitions[top]doc := s.pop()- if p.skipped < p.offset {+ fresh, err := p.firstSighting(doc.Payload)+ if err != nil {+ return err+ }+ switch {+ case !fresh:+ // DISTINCT already emitted this payload.+ case p.skipped < p.offset:p.skipped++- } else {+ default:// Defensive copy — json.RawMessage aliases the underlying buffer.p.pending = append(p.pending, append([]byte(nil), doc.Payload...))p.emitted++@@ -346,9 +361,28 @@ func (p *orderByPipeline) merge() {continue}if !p.fanout.drained(s.id) {- return+ return nil}}+ return nil+}++// firstSighting reports whether a DISTINCT lets this payload through, and+// records it. Always true for a plan without a DISTINCT.+func (p *orderByPipeline) firstSighting(payload json.RawMessage) (bool, error) {+ if p.seen == nil {+ return true, nil+ }+ canonical, err := canonicalJSON(payload)+ if err != nil {+ return false, err+ }+ key := string(canonical)+ if _, ok := p.seen[key]; ok {+ return false, nil+ }+ p.seen[key] = struct{}{}+ return true, nil}// waitingForPage reports whether a range has nothing buffered and pages stilldiff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.goindex 7a3d3cb69e..a886d62c12 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go@@ -501,3 +501,88 @@ func asStrings(rows [][]byte) []string {}return out}++// ------------------------------------------------- TOP and DISTINCT composed++func newTopOrderByPipeline(t *testing.T, n int, pkRangeIDs ...string) *orderByPipeline {+ t.Helper()+ plan := &planDoc{}+ plan.QueryInfo.OrderBy = []string{"Ascending"}+ plan.QueryInfo.Top = intPtr(n)+ plan.QueryInfo.RewrittenQuery = `SELECT TOP 5 c._rid, [{"item":c.x}] AS orderByItems, c AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x ASC`+ p, err := newOrderByPipeline(plan, pkRangeIDs)+ require.NoError(t, err)+ return p+}++func newDistinctOrderByPipeline(t *testing.T, pkRangeIDs ...string) *orderByPipeline {+ t.Helper()+ plan := &planDoc{}+ plan.QueryInfo.OrderBy = []string{"Ascending"}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.RewrittenQuery = `SELECT c._rid, [{"item":c.x}] AS orderByItems, {"x":c.x} AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x`+ p, err := newOrderByPipeline(plan, pkRangeIDs)+ require.NoError(t, err)+ return p+}++// The per-partition query carries TOP n, so the merge sees up to n rows from+// every range and the cap has to be applied again over the merged stream.+func TestOrderByPipeline_TopCapsTheMergedStream(t *testing.T) {+ p := newTopOrderByPipeline(t, 3, "0", "1")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `1`, payload: `{"x":1}`},+ ordered{ob: `4`, payload: `{"x":4}`},+ ordered{ob: `6`, payload: `{"x":6}`},+ ),+ "1": mkOrderByBody(t,+ ordered{ob: `2`, payload: `{"x":2}`},+ ordered{ob: `3`, payload: `{"x":3}`},+ ordered{ob: `5`, payload: `{"x":5}`},+ ),+ })+ assert.Equal(t, []string{`{"x":1}`, `{"x":2}`, `{"x":3}`}, rows)+}++// DISTINCT with an ORDER BY dedups the merged stream: the service reports+// distinctType "Unordered" for the object form even under ORDER BY, and the+// per-partition query carries no DISTINCT at all, so every duplicate arrives.+func TestOrderByPipeline_DistinctDedupsTheMergedStream(t *testing.T) {+ p := newDistinctOrderByPipeline(t, "0", "1")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"c"`, payload: `{"x":"c"}`},+ ),+ "1": mkOrderByBody(t,+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"b"`, payload: `{"x":"b"}`},+ ordered{ob: `"c"`, payload: `{"x":"c"}`},+ ),+ })+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`, `{"x":"c"}`}, rows)+}++// A duplicate must not spend a row of the LIMIT, or a paged DISTINCT comes+// back short.+func TestOrderByPipeline_DistinctDropsDuplicatesBeforeTheLimit(t *testing.T) {+ plan := &planDoc{}+ plan.QueryInfo.OrderBy = []string{"Ascending"}+ plan.QueryInfo.DistinctType = "Unordered"+ plan.QueryInfo.Limit = intPtr(2)+ plan.QueryInfo.RewrittenQuery = `SELECT c._rid, [{"item":c.x}] AS orderByItems, {"x":c.x} AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x OFFSET 0 LIMIT 2`+ p, err := newOrderByPipeline(plan, []string{"0"})+ require.NoError(t, err)++ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `"b"`, payload: `{"x":"b"}`},+ ),+ })+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`}, rows)+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go b/backend/plugin/db/cosmosdb/queryengine/gonative/partition.goindex 6dee2a3904..70b5e23b1d 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go@@ -13,7 +13,8 @@ import (// VALUE form — plan.QueryInfo.Aggregates non-empty:// {"Documents":[[{"item":18}]]}// Each Documents[i] is an array of one item-object per aggregate in-// plan.QueryInfo.Aggregates (positional match).+// plan.QueryInfo.Aggregates (positional match). MIN/MAX carry item2 here+// too: {"item":1,"item2":{"min":1,"count":5}}.//// Aliased form — plan.QueryInfo.GroupByAliasToAggregateType non-empty:// {"Documents":[{"payload":{"alias":{"item":...,"item2":{...}}}}]}@@ -21,15 +22,18 @@ import (// keyed by alias name. Item2 is present for MIN/MAX (reliable partial// {min|max, count}); for COUNT/SUM/AVG only Item is populated.-// valueItem is each element inside a VALUE-form partition document.-type valueItem struct {- Item json.RawMessage `json:"item"`+// aggItem is one aggregate's partial inside a partition document. Item is the+// aggregate's own value; Item2 is the cross-partition partial the gateway adds+// for MIN and MAX, which carries the count of rows the partition contributed.+type aggItem struct {+ Item json.RawMessage `json:"item"`+ Item2 json.RawMessage `json:"item2,omitempty"`}// valuePartitionResponse decodes VALUE-form per-partition bodies.// Documents[i] is an array whose length equals the number of aggregates.type valuePartitionResponse struct {- Documents [][]valueItem `json:"Documents"`+ Documents [][]aggItem `json:"Documents"`}// parseValuePartitionResponse decodes a per-partition response body for a@@ -42,17 +46,9 @@ func parseValuePartitionResponse(raw []byte) (*valuePartitionResponse, error) {return &r, nil}-// aliasedItem is the per-alias partial inside an aliased-form partition-// document's payload. Item is always present; Item2 is populated only for-// MIN/MAX and carries the reliable cross-partition partial.-type aliasedItem struct {- Item json.RawMessage `json:"item"`- Item2 json.RawMessage `json:"item2,omitempty"`-}-// aliasedDoc is one entry of Documents for an aliased aggregate query.type aliasedDoc struct {- Payload map[string]aliasedItem `json:"payload"`+ Payload map[string]aggItem `json:"payload"`}// aliasedPartitionResponse decodes aliased-form per-partition bodies.diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.goindex 372cef18c5..2d0546546c 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go@@ -2,7 +2,6 @@ package gonativeimport ("encoding/json"- "fmt""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine""github.com/pkg/errors"@@ -20,8 +19,16 @@ type aggPipelineBase struct {rewrittenQuery stringfanout rangeFanoutaggregators []aggregator // len == number of aggregates to emit+ kinds []string // positional: aggregate kind per aggregatoremitted boolclosed bool++ // skip and take carry the plan's OFFSET and LIMIT. An aggregate query+ // produces exactly one row, so they can only keep it or drop it; the+ // gateway strips both from the rewritten query and leaves them here.+ // take is negative when the plan sets no LIMIT.+ skip int+ take int}// requests returns the work to issue next: the initial fan-out across every@@ -30,6 +37,39 @@ func (b *aggPipelineBase) requests() []queryengine.QueryRequest {return b.fanout.requests(b.rewrittenQuery)}+// keepsRow reports whether the plan's OFFSET/LIMIT leaves the single+// aggregate row in the result: OFFSET 1 skips past it and LIMIT 0 asks for+// nothing, and the gateway answers both with no rows at all.+func (b *aggPipelineBase) keepsRow() bool {+ return b.skip == 0 && b.take != 0+}++// newAggPipelineBase builds the state both aggregate pipelines share from the+// plan: the per-partition query, one aggregator per kind in positional order,+// and the plan's OFFSET/LIMIT.+func newAggPipelineBase(plan *planDoc, pkRangeIDs []string, kinds []string) (aggPipelineBase, error) {+ aggs := make([]aggregator, len(kinds))+ for i, kind := range kinds {+ a := newAggregator(kind)+ if a == nil {+ return aggPipelineBase{}, errors.Errorf("gonative: unsupported aggregate %q", kind)+ }+ aggs[i] = a+ }+ skip, take, err := planSkipTake(plan.QueryInfo)+ if err != nil {+ return aggPipelineBase{}, err+ }+ return aggPipelineBase{+ rewrittenQuery: plan.QueryInfo.RewrittenQuery,+ fanout: newRangeFanout(pkRangeIDs),+ aggregators: aggs,+ kinds: kinds,+ skip: skip,+ take: take,+ }, nil+}+// -----------------------------------------------------------------------------// VALUE-form pipeline — plan.QueryInfo.Aggregates non-empty, hasSelectValue true.// Emits one final scalar (single aggregate) or one JSON array (multi-aggregate).@@ -42,21 +82,11 @@ type valuePipeline struct {// Returns an error if the plan references an aggregate kind the engine cannot// handle, so the caller can surface ErrUnsupportedPlanFeature.func newValuePipeline(plan *planDoc, pkRangeIDs []string) (*valuePipeline, error) {- aggs := make([]aggregator, len(plan.QueryInfo.Aggregates))- for i, kind := range plan.QueryInfo.Aggregates {- a := newAggregator(kind)- if a == nil {- return nil, errors.Errorf("gonative: unsupported aggregate %q", kind)- }- aggs[i] = a+ base, err := newAggPipelineBase(plan, pkRangeIDs, plan.QueryInfo.Aggregates)+ if err != nil {+ return nil, err}- return &valuePipeline{- aggPipelineBase: aggPipelineBase{- rewrittenQuery: plan.QueryInfo.RewrittenQuery,- fanout: newRangeFanout(pkRangeIDs),- aggregators: aggs,- },- }, nil+ return &valuePipeline{aggPipelineBase: base}, nil}// Query returns the gateway-rewritten per-partition query.@@ -90,6 +120,9 @@ func (p *valuePipeline) Run() (*queryengine.PipelineResult, error) {return nil, err}p.emitted = true+ if item == nil || !p.keepsRow() {+ return &queryengine.PipelineResult{IsCompleted: true}, nil+ }return &queryengine.PipelineResult{Items: [][]byte{item}, IsCompleted: true}, nil}@@ -105,7 +138,7 @@ func (p *valuePipeline) ProvideData(results []queryengine.QueryResult) error {return errors.Errorf("gonative: value partition document has %d items, expected %d", len(doc), len(p.aggregators))}for i, item := range doc {- if err := p.aggregators[i].feedPartial(item.Item); err != nil {+ if err := feedAggregatePartial(p.aggregators[i], p.kinds[i], item); err != nil {return err}}@@ -121,6 +154,11 @@ func (p *valuePipeline) ProvideData(results []queryengine.QueryResult) error {// For single-aggregate queries it is the finalized scalar itself; for// multi-aggregate queries (plan.QueryInfo.Aggregates with len > 1) it is a// JSON array of finalized scalars in positional order.+//+// Returns nil when a single aggregate finalized to undefined — MIN, MAX or AVG+// over an empty set — because the gateway answers those with no row. An+// undefined slot inside a multi-aggregate array has no such spelling and goes+// out as null.func (p *valuePipeline) finalRow() ([]byte, error) {finals := make([]json.RawMessage, len(p.aggregators))for i, a := range p.aggregators {@@ -128,6 +166,12 @@ func (p *valuePipeline) finalRow() ([]byte, error) {if err != nil {return nil, err}+ if v == nil {+ if len(p.aggregators) == 1 {+ return nil, nil+ }+ v = json.RawMessage(`null`)+ }finals[i] = v}if len(finals) == 1 {@@ -143,7 +187,6 @@ func (p *valuePipeline) finalRow() ([]byte, error) {type aliasedPipeline struct {aggPipelineBasealiases []string // preserved order from plan.QueryInfo.GroupByAliases- kinds []string // positional: aggregate kind per alias}func newAliasedPipeline(plan *planDoc, pkRangeIDs []string) (*aliasedPipeline, error) {@@ -152,28 +195,18 @@ func newAliasedPipeline(plan *planDoc, pkRangeIDs []string) (*aliasedPipeline, e}aliases := append([]string(nil), plan.QueryInfo.GroupByAliases...)kinds := make([]string, len(aliases))- aggs := make([]aggregator, len(aliases))for i, alias := range aliases {kind, ok := plan.QueryInfo.GroupByAliasToAggregateType[alias]if !ok {return nil, errors.Errorf("gonative: alias %q missing from groupByAliasToAggregateType", alias)}- a := newAggregator(kind)- if a == nil {- return nil, errors.Errorf("gonative: unsupported aggregate %q for alias %q", kind, alias)- }kinds[i] = kind- aggs[i] = a}- return &aliasedPipeline{- aggPipelineBase: aggPipelineBase{- rewrittenQuery: plan.QueryInfo.RewrittenQuery,- fanout: newRangeFanout(pkRangeIDs),- aggregators: aggs,- },- aliases: aliases,- kinds: kinds,- }, nil+ base, err := newAggPipelineBase(plan, pkRangeIDs, kinds)+ if err != nil {+ return nil, err+ }+ return &aliasedPipeline{aggPipelineBase: base, aliases: aliases}, nil}func (p *aliasedPipeline) Query() string { return p.rewrittenQuery }@@ -195,6 +228,9 @@ func (p *aliasedPipeline) Run() (*queryengine.PipelineResult, error) {return nil, err}p.emitted = true+ if !p.keepsRow() {+ return &queryengine.PipelineResult{IsCompleted: true}, nil+ }return &queryengine.PipelineResult{Items: [][]byte{item}, IsCompleted: true}, nil}@@ -210,7 +246,7 @@ func (p *aliasedPipeline) ProvideData(results []queryengine.QueryResult) error {if !ok {return errors.Errorf("gonative: aliased partition missing alias %q in payload", alias)}- if err := p.feedAliasedPartial(slot, p.kinds[slot], partial); err != nil {+ if err := feedAggregatePartial(p.aggregators[slot], p.kinds[slot], partial); err != nil {return err}}@@ -222,32 +258,11 @@ func (p *aliasedPipeline) ProvideData(results []queryengine.QueryResult) error {return nil}-// feedAliasedPartial routes an aliasedItem to the right aggregator. For-// Min/Max it prefers Item2 (the reliable `{min|max, count}` partial) when-// present, falling back to wrapping Item when the gateway short-circuited to-// a single-partition shape (Item2 absent).-func (p *aliasedPipeline) feedAliasedPartial(slot int, kind string, partial aliasedItem) error {- switch kind {- case "Min":- if len(partial.Item2) > 0 {- return p.aggregators[slot].feedPartial(partial.Item2)- }- synth := fmt.Appendf(nil, `{"min":%s,"count":1}`, partial.Item)- return p.aggregators[slot].feedPartial(synth)- case "Max":- if len(partial.Item2) > 0 {- return p.aggregators[slot].feedPartial(partial.Item2)- }- synth := fmt.Appendf(nil, `{"max":%s,"count":1}`, partial.Item)- return p.aggregators[slot].feedPartial(synth)- default:- // Count, Sum, Avg — the Item itself carries the partial in the right shape.- return p.aggregators[slot].feedPartial(partial.Item)- }-}-// finalRow builds `{"<alias1>": <v1>, ...}` preserving the original alias order// from the plan (matters for deterministic output — map iteration is random).+// An alias whose aggregate finalized to undefined is left out of the object,+// which is how the gateway spells MIN, MAX or AVG over an empty set: the whole+// row comes back as `{}`.func (p *aliasedPipeline) finalRow() ([]byte, error) {finals := make([]json.RawMessage, len(p.aggregators))for i, a := range p.aggregators {@@ -262,7 +277,10 @@ func (p *aliasedPipeline) finalRow() ([]byte, error) {var buf []bytebuf = append(buf, '{')for i, alias := range p.aliases {- if i > 0 {+ if finals[i] == nil {+ continue+ }+ if len(buf) > 1 {buf = append(buf, ',')}keyRaw, err := json.Marshal(alias)diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.goindex 203fc34984..c994f640ec 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go@@ -5,6 +5,8 @@ import ("fmt""os""path/filepath"+ "slices"+ "strings""testing""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"@@ -219,7 +221,7 @@ func TestAggPipeline_AliasedCountFoldsPartialsAcrossPages(t *testing.T) {// drivePipelineAll runs a pipeline end-to-end against captured fixtures and// returns every emitted row. Used for operators that emit more than one row-// (DISTINCT, later Stage 3+ ORDER BY/TOP/LIMIT/GROUP BY).+// (DISTINCT, ORDER BY, TOP, OFFSET/LIMIT, GROUP BY).func drivePipelineAll(t *testing.T, queryID, originalQuery string) []json.RawMessage {t.Helper()base := filepath.Join("testdata", "query_"+queryID)@@ -416,3 +418,109 @@ func indexOf(haystack, needle []byte) int {}return -1}++// drivePlan runs a recorded plan against the given per-range response bodies+// and returns every emitted row. Unlike drivePipelineAll the bodies are given+// inline, which is how the cases below pin an answer against a partition that+// matched no row.+func drivePlan(t *testing.T, planFile, query string, bodies map[string][]byte) []json.RawMessage {+ t.Helper()+ plan := mustReadFile(t, filepath.Join("testdata", "query_"+planFile+"_plan.json"))+ ids := make([]string, 0, len(bodies))+ for id := range bodies {+ ids = append(ids, fmt.Sprintf("{\"id\":%q}", id))+ }+ // Range order decides which rows a window lands on, so it must not come+ // from map iteration.+ slices.Sort(ids)+ pkranges := fmt.Sprintf(`{"PartitionKeyRanges":[%s]}`, strings.Join(ids, ","))++ pipe, err := gonative.Default().CreateQueryPipeline(query, string(plan), pkranges)+ require.NoError(t, err, "CreateQueryPipeline")+ defer pipe.Close()++ var rows []json.RawMessage+ for i := 0; i < 50 && !pipe.IsComplete(); i++ {+ res, err := pipe.Run()+ require.NoError(t, err, "Run turn %d", i)+ if len(res.Requests) > 0 {+ results := make([]queryengine.QueryResult, 0, len(res.Requests))+ for _, req := range res.Requests {+ body, ok := bodies[req.PartitionKeyRangeID]+ require.True(t, ok, "missing body for pk-range %s", req.PartitionKeyRangeID)+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, body, ""))+ }+ require.NoError(t, pipe.ProvideData(results))+ }+ for _, it := range res.Items {+ rows = append(rows, append(json.RawMessage(nil), it...))+ }+ }+ require.True(t, pipe.IsComplete(), "pipeline did not converge")+ return rows+}++// A VALUE-form MIN reads its partial from item2, the only one that reports how+// many rows the partition contributed. Reading item instead fed a bare scalar+// to an aggregator expecting `{min, count}`.+func TestValuePipeline_MinReadsTheItem2Partial(t *testing.T) {+ rows := drivePlan(t, "min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 0`,+ map[string][]byte{+ "0": []byte(`{"Documents":[[{"item":5,"item2":{"min":5,"count":3}}]]}`),+ "1": []byte(`{"Documents":[[{"item":2,"item2":{"min":2,"count":7}}]]}`),+ })+ require.Len(t, rows, 1)+ assert.JSONEq(t, `2`, string(rows[0]))+}++// MIN over a result set no row matched is undefined, and the gateway answers+// it with no row at all. Before this the absent partial failed the query with+// "gonative/agg.min: unexpected end of JSON input".+func TestValuePipeline_MinOverAnEmptySetEmitsNoRow(t *testing.T) {+ rows := drivePlan(t, "min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 99999999`,+ map[string][]byte{+ "0": []byte(`{"Documents":[[{"item2":{"count":0}}]]}`),+ "1": []byte(`{"Documents":[[{"item2":{"count":0}}]]}`),+ })+ assert.Empty(t, rows)+}++// The aliased form of the same query answers with one row, the alias left out.+func TestAliasedPipeline_MinOverAnEmptySetOmitsTheAlias(t *testing.T) {+ rows := drivePlan(t, "min_aliased_empty", `SELECT MIN(c.seq) AS m FROM c WHERE c.seq > 99999999`,+ map[string][]byte{+ "0": []byte(`{"Documents":[{"payload":{"m":{"item2":{"count":0}}}}]}`),+ "1": []byte(`{"Documents":[{"payload":{"m":{"item2":{"count":0}}}}]}`),+ })+ require.Len(t, rows, 1)+ assert.JSONEq(t, `{}`, string(rows[0]))+}++// An aggregate produces exactly one row, so OFFSET 1 skips past it and the+// gateway answers with nothing.+func TestValuePipeline_OffsetPastTheOnlyRowEmitsNothing(t *testing.T) {+ rows := drivePlan(t, "agg_offset_limit", `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`,+ map[string][]byte{+ "0": []byte(`{"Documents":[[{"item":4}]]}`),+ "1": []byte(`{"Documents":[[{"item":6}]]}`),+ })+ assert.Empty(t, rows)+}++// OFFSET/LIMIT with no ORDER BY cuts one window out of the concatenated+// ranges. The engine used to reject the plan outright.+func TestSkipTakePipeline_OffsetLimitWithoutOrderBy(t *testing.T) {+ rows := drivePlan(t, "offset_limit", `SELECT c.seq FROM c OFFSET 2 LIMIT 3`,+ map[string][]byte{+ "0": []byte(`{"Documents":[{"seq":1},{"seq":2},{"seq":3}]}`),+ })+ assert.Equal(t, []string{`{"seq":3}`}, jsonStrings(rows))+}++func jsonStrings(rows []json.RawMessage) []string {+ out := make([]string, 0, len(rows))+ for _, r := range rows {+ out = append(out, string(r))+ }+ return out+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go b/backend/plugin/db/cosmosdb/queryengine/gonative/plan.goindex 9f544312e3..8d6fc7ae1e 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go@@ -6,10 +6,7 @@ import ("github.com/pkg/errors")-// planDoc is the subset of the Cosmos query-plan document that Stage 1 uses.-// Later stages (DISTINCT, ORDER BY, TOP, OFFSET/LIMIT, GROUP BY) will read-// additional fields; declaring them defensively here keeps Stage 1 parsing-// tolerant of plans that carry extra information.+// planDoc is the subset of the Cosmos query-plan document the engine reads.//// Observed shapes (see queryengine/gonative/testdata)://@@ -30,22 +27,25 @@ type planDoc struct {}// planQueryInfo holds the instruction set the engine uses to build a pipeline.+//+// Every field here is an instruction, so the pipeline the dispatch picks must+// claim each one the plan sets — see requireHandled. A field added to this+// struct is therefore rejected until a pipeline claims it, which is what keeps+// a new plan shape from losing an operator silently.type planQueryInfo struct {RewrittenQuery string `json:"rewrittenQuery"`HasSelectValue bool `json:"hasSelectValue"`Aggregates []string `json:"aggregates"`GroupByAliases []string `json:"groupByAliases"`GroupByAliasToAggregateType map[string]string `json:"groupByAliasToAggregateType"`- // Fields that Stage 1 does not act on but decodes defensively so the- // dispatch logic can reject unsupported plans cleanly.- DistinctType string `json:"distinctType,omitempty"`- Top *int `json:"top,omitempty"`- Offset *int `json:"offset,omitempty"`- Limit *int `json:"limit,omitempty"`- OrderBy []string `json:"orderBy,omitempty"`- OrderByExpressions []string `json:"orderByExpressions,omitempty"`- GroupByExpressions []string `json:"groupByExpressions,omitempty"`- HasNonStreamingOrderBy bool `json:"hasNonStreamingOrderBy,omitempty"`+ DistinctType string `json:"distinctType,omitempty"`+ Top *int `json:"top,omitempty"`+ Offset *int `json:"offset,omitempty"`+ Limit *int `json:"limit,omitempty"`+ OrderBy []string `json:"orderBy,omitempty"`+ OrderByExpressions []string `json:"orderByExpressions,omitempty"`+ GroupByExpressions []string `json:"groupByExpressions,omitempty"`+ HasNonStreamingOrderBy bool `json:"hasNonStreamingOrderBy,omitempty"`}// queryRange mirrors the gateway's pk-range description inside the plan.@@ -56,15 +56,47 @@ type queryRange struct {IsMaxInclusive bool `json:"isMaxInclusive"`}-// parsePlan decodes the gateway's query-plan JSON payload.+// parsePlan decodes the gateway's query-plan JSON payload. It normalizes the+// gateway's "None" distinctType to the empty string, so every caller can read+// a set DistinctType as "this query has a DISTINCT".func parsePlan(raw []byte) (*planDoc, error) {var p planDocif err := json.Unmarshal(raw, &p); err != nil {return nil, errors.Wrap(err, "gonative: parse query plan")}+ if p.QueryInfo.DistinctType == "None" {+ p.QueryInfo.DistinctType = ""+ }return &p, nil}+// planSkipTake reads the plan's OFFSET, LIMIT and TOP as one global skip and+// take, which is what every pipeline applies them as: the gateway strips them+// from the rewritten query (or widens LIMIT to OFFSET+LIMIT) precisely so the+// client applies them once, over the merged result.+//+// take is negative when the query names no upper bound. No Cosmos syntax+// combines TOP with OFFSET/LIMIT, but a plan carrying both can only mean the+// tighter of the two.+func planSkipTake(qi planQueryInfo) (skip int, take int, err error) {+ take = -1+ for name, v := range map[string]*int{"OFFSET": qi.Offset, "LIMIT": qi.Limit, "TOP": qi.Top} {+ if v != nil && *v < 0 {+ return 0, 0, errors.Errorf("gonative: negative %s=%d", name, *v)+ }+ }+ if qi.Offset != nil {+ skip = *qi.Offset+ }+ if qi.Limit != nil {+ take = *qi.Limit+ }+ if qi.Top != nil && (take < 0 || *qi.Top < take) {+ take = *qi.Top+ }+ return skip, take, nil+}+// pkRangesResponse decodes the gateway's pk-ranges response just enough to// extract the ordered partition-key-range IDs the pipeline needs to fan out.type pkRangesResponse struct {diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.jsonnew file mode 100644index 0000000000..3ac05c3402--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.json@@ -0,0 +1,30 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": 1,+ "limit": 1,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [+ "Count"+ ],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT VALUE [{\"item\": COUNT(1)}]\nFROM c",+ "hasSelectValue": true,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.jsonnew file mode 100644index 0000000000..435c72f251--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.json@@ -0,0 +1,28 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "Unordered",+ "top": null,+ "offset": 2,+ "limit": 3,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT DISTINCT c.bucket\nFROM c",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.jsonnew file mode 100644index 0000000000..1597ef20a9--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.json@@ -0,0 +1,32 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "Unordered",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [+ "Ascending"+ ],+ "orderByExpressions": [+ "c.bucket"+ ],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT c._rid, [{\"item\": c.bucket}] AS orderByItems, {\"bucket\": c.bucket} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.bucket",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.jsonnew file mode 100644index 0000000000..420a174811--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.json@@ -0,0 +1,32 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "Ordered",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [+ "Ascending"+ ],+ "orderByExpressions": [+ "c.bucket"+ ],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT c._rid, [{\"item\": c.bucket}] AS orderByItems, c.bucket AS payload\nFROM c\nWHERE (({documentdb-formattableorderbyquery-filter}) AND IS_DEFINED(c.bucket))\nORDER BY c.bucket",+ "hasSelectValue": true,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.jsonnew file mode 100644index 0000000000..da34c5a65e--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.json@@ -0,0 +1,28 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "Unordered",+ "top": 3,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT DISTINCT c.bucket\nFROM c",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.jsonnew file mode 100644index 0000000000..0095c198e5--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.json@@ -0,0 +1,28 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "Unordered",+ "top": null,+ "offset": 0,+ "limit": 3,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT DISTINCT VALUE c.bucket\nFROM c",+ "hasSelectValue": true,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.jsonnew file mode 100644index 0000000000..4c68e90482--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.json@@ -0,0 +1,36 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": 1,+ "limit": 2,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [+ "c.bucket"+ ],+ "groupByAliases": [+ "bucket",+ "n"+ ],+ "aggregates": [],+ "groupByAliasToAggregateType": {+ "bucket": null,+ "n": "Count"+ },+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.jsonnew file mode 100644index 0000000000..260b3a4ea2--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.json@@ -0,0 +1,36 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [+ "c.bucket"+ ],+ "groupByAliases": [+ "bucket",+ "n"+ ],+ "aggregates": [],+ "groupByAliasToAggregateType": {+ "bucket": null,+ "n": "Count"+ },+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket\nORDER BY c.bucket",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.jsonnew file mode 100644index 0000000000..26af6f2a8a--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.json@@ -0,0 +1,36 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [+ "c.bucket"+ ],+ "groupByAliases": [+ "bucket",+ "n"+ ],+ "aggregates": [],+ "groupByAliasToAggregateType": {+ "bucket": null,+ "n": "Count"+ },+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.jsonnew file mode 100644index 0000000000..18ab1b7cd8--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.json@@ -0,0 +1,36 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": 3,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [+ "c.bucket"+ ],+ "groupByAliases": [+ "bucket",+ "n"+ ],+ "aggregates": [],+ "groupByAliasToAggregateType": {+ "bucket": null,+ "n": "Count"+ },+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.jsonnew file mode 100644index 0000000000..2bcbfdc8f0--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.json@@ -0,0 +1,32 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [+ "m"+ ],+ "aggregates": [],+ "groupByAliasToAggregateType": {+ "m": "Min"+ },+ "rewrittenQuery": "SELECT {\"m\": {\"item\": MIN(c.seq), \"item2\": {\"min\": MIN(c.seq), \"count\": COUNT(c.seq)}}} AS payload\nFROM c\nWHERE (c.seq > 99999999)",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.jsonnew file mode 100644index 0000000000..672b1f5f40--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.json@@ -0,0 +1,30 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": null,+ "limit": null,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [+ "Min"+ ],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT VALUE [{\"item\": MIN(c.seq), \"item2\": {\"min\": MIN(c.seq), \"count\": COUNT(c.seq)}}]\nFROM c\nWHERE (c.seq > 99999999)",+ "hasSelectValue": true,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.jsonnew file mode 100644index 0000000000..6be77dd13f--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.json@@ -0,0 +1,28 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": null,+ "offset": 2,+ "limit": 3,+ "orderBy": [],+ "orderByExpressions": [],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT c.seq\nFROM c\nOFFSET 0 LIMIT 5",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.jsonnew file mode 100644index 0000000000..971941a48b--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.json@@ -0,0 +1,32 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": 5,+ "offset": null,+ "limit": null,+ "orderBy": [+ "Ascending"+ ],+ "orderByExpressions": [+ "c.seq"+ ],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT TOP 5 c._rid, [{\"item\": c.seq}] AS orderByItems, {\"seq\": c.seq} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.seq ASC",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.jsonnew file mode 100644index 0000000000..a04dc9d7f4--- /dev/null+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.json@@ -0,0 +1,32 @@+{+ "partitionedQueryExecutionInfoVersion": 2,+ "queryInfo": {+ "distinctType": "None",+ "top": 5,+ "offset": null,+ "limit": null,+ "orderBy": [+ "Descending"+ ],+ "orderByExpressions": [+ "c.seq"+ ],+ "groupByExpressions": [],+ "groupByAliases": [],+ "aggregates": [],+ "groupByAliasToAggregateType": {},+ "rewrittenQuery": "SELECT TOP 5 c._rid, [{\"item\": c.seq}] AS orderByItems, {\"seq\": c.seq} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.seq DESC",+ "hasSelectValue": false,+ "dCountInfo": null,+ "hasNonStreamingOrderBy": false+ },+ "queryRanges": [+ {+ "min": "",+ "max": "FF",+ "isMinInclusive": true,+ "isMaxInclusive": false+ }+ ],+ "hybridSearchQueryInfo": null+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.goindex bf5fba17e1..160238df89 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go@@ -7,10 +7,10 @@ import ("github.com/pkg/errors")-// topPartitionResponse decodes a "no rewrite" per-partition response — the+// topPartitionResponse decodes an unwrapped per-partition response — the// Documents array carries raw user-shaped rows directly (no orderByItems /-// payload wrapping). Used for standalone TOP, where the gateway runs the-// original query per partition and returns whatever matches.+// payload wrapping). Used for TOP and for OFFSET/LIMIT without an ORDER BY,+// where each partition returns whatever matches and the engine cuts the page.type topPartitionResponse struct {Documents []json.RawMessage `json:"Documents"`}@@ -23,68 +23,76 @@ func parseTopPartitionResponse(raw []byte) (*topPartitionResponse, error) {return &r, nil}-// topPipeline implements queryengine.QueryPipeline for standalone TOP queries-// — plans with `top: N` and none of ORDER BY, DISTINCT, aggregates, or GROUP BY.-// Rows from each partition are concatenated and the first N are emitted.+// skipTakePipeline implements queryengine.QueryPipeline for the plans whose+// only operators cut a window out of an unordered result: `TOP n`, and+// `OFFSET n LIMIT m` with no ORDER BY. Rows from each partition are+// concatenated, the first `skip` are dropped and the next `take` emitted.//-// TOP composed with ORDER BY/DISTINCT/aggregates/GROUP BY is out of Stage 4's-// scope; the dispatch rejects those plans with ErrUnsupportedPlanFeature.+// Which rows those are is unspecified without an ORDER BY, which is what+// makes the window safe to cut here: the gateway widens the per-partition+// query to `OFFSET 0 LIMIT (offset+limit)` (or leaves TOP n on it), so every+// range offers at least as many rows as the window needs and no row the+// window should hold is missing from the union.//-// A TOP larger than a response page needs several pages from a range. The-// pipeline keeps asking for them until it holds N rows, at which point it+// A window larger than a response page needs several pages from a range. The+// pipeline keeps asking for them until it holds take rows, at which point it// stops: IsComplete trips on the emitted count and no further page is// requested, however many rows the ranges still hold.-type topPipeline struct {+type skipTakePipeline struct {query string- n int+ skip int+ take int // negative means unlimitedfanout rangeFanoutclosed bool- // pending items ready to flush to the caller, already capped at N total.+ // pending items ready to flush to the caller, already capped at take.pending [][]byte+ // skipped counts the rows dropped by skip so far.+ skipped int// emitted tracks how many rows have been drained via Run(); we stop- // accepting/emitting once we hit N.+ // accepting/emitting once we hit take.emitted int// accumulated counts pending + emitted. Keeps ProvideData from growing- // `pending` past N so downstream flushes never over-emit.+ // `pending` past take so downstream flushes never over-emit.accumulated int}-// newTopPipeline builds a pipeline for a plan whose only operator is TOP.-// Returns ErrUnsupportedPlanFeature when the plan carries any other operator-// that would require composition (ORDER BY, DISTINCT, aggregate, GROUP BY).-func newTopPipeline(plan *planDoc, pkRangeIDs []string) (*topPipeline, error) {- if plan.QueryInfo.Top == nil {- return nil, errors.New("gonative: topPipeline requires a non-nil Top")+// newSkipTakePipeline builds a pipeline for a plan whose only operators are+// TOP and/or OFFSET/LIMIT. Composition with another operator is the+// dispatch's call, not this constructor's: CreateQueryPipeline rejects a plan+// carrying a field this pipeline cannot honour before it gets here.+func newSkipTakePipeline(plan *planDoc, pkRangeIDs []string, query string) (*skipTakePipeline, error) {+ if plan.QueryInfo.Top == nil && plan.QueryInfo.Offset == nil && plan.QueryInfo.Limit == nil {+ return nil, errors.New("gonative: skipTakePipeline requires TOP, OFFSET or LIMIT")}- // Composition with other operators is a follow-up. Stage 4 only handles- // standalone TOP — the dispatcher routes composed plans elsewhere or- // rejects them.- if len(plan.QueryInfo.OrderBy) > 0 ||- plan.QueryInfo.DistinctType == "Unordered" || plan.QueryInfo.DistinctType == "Ordered" ||- len(plan.QueryInfo.Aggregates) > 0 ||- len(plan.QueryInfo.GroupByAliasToAggregateType) > 0 ||- len(plan.QueryInfo.GroupByExpressions) > 0 {- return nil, ErrUnsupportedPlanFeature+ skip, take, err := planSkipTake(plan.QueryInfo)+ if err != nil {+ return nil, err}- n := *plan.QueryInfo.Top- if n < 0 {- return nil, errors.Errorf("gonative: negative TOP N=%d", n)+ // A bare TOP plan carries no rewrittenQuery, so each range answers the+ // user's own query, TOP and all. An OFFSET/LIMIT plan carries one widened+ // to `OFFSET 0 LIMIT (offset+limit)`, and running that is what lets this+ // pipeline cut the window globally rather than per range.+ if plan.QueryInfo.RewrittenQuery != "" {+ query = plan.QueryInfo.RewrittenQuery}- return &topPipeline{- n: n,+ return &skipTakePipeline{+ query: query,+ skip: skip,+ take: take,fanout: newRangeFanout(pkRangeIDs),}, nil}// Query implements queryengine.QueryPipeline.-func (p *topPipeline) Query() string { return p.query }+func (p *skipTakePipeline) Query() string { return p.query }-// IsComplete returns true once either (a) we've emitted N rows — which covers-// TOP 0, so the pipeline never fires a request for it — or (b) every range has-// delivered its last page and all pending rows have been flushed.-func (p *topPipeline) IsComplete() bool {- if p.emitted >= p.n {+// IsComplete returns true once either (a) we've emitted take rows — which+// covers TOP 0 and LIMIT 0, so the pipeline never fires a request for them —+// or (b) every range has delivered its last page and all pending rows have+// been flushed.+func (p *skipTakePipeline) IsComplete() bool {+ if p.take >= 0 && p.emitted >= p.take {return true}if len(p.pending) > 0 {@@ -94,12 +102,12 @@ func (p *topPipeline) IsComplete() bool {}// Close implements queryengine.QueryPipeline.-func (p *topPipeline) Close() { p.closed = true }+func (p *skipTakePipeline) Close() { p.closed = true }// Run drives one pipeline turn: flush the rows already in hand, then ask for// the next page of every range that has one — the initial fan-out on the first// call.-func (p *topPipeline) Run() (*queryengine.PipelineResult, error) {+func (p *skipTakePipeline) Run() (*queryengine.PipelineResult, error) {if len(p.pending) > 0 {batch := p.pendingp.pending = nil@@ -112,14 +120,14 @@ func (p *topPipeline) Run() (*queryengine.PipelineResult, error) {return &queryengine.PipelineResult{Requests: p.fanout.requests(p.query)}, nil}-// ProvideData consumes each partition's response and appends rows to `pending`-// up to the overall TOP N limit. Anything beyond N is dropped.-func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {+// ProvideData consumes each partition's response, drops the first `skip` rows+// and appends the rest to `pending` up to `take` rows in all.+func (p *skipTakePipeline) ProvideData(results []queryengine.QueryResult) error {for _, r := range results {if err := p.fanout.note(r); err != nil {return err}- if p.accumulated >= p.n {+ if p.full() {// Already have enough rows — leave this response unread.continue}@@ -128,9 +136,13 @@ func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {return err}for _, doc := range resp.Documents {- if p.accumulated >= p.n {+ if p.full() {break}+ if p.skipped < p.skip {+ p.skipped+++ continue+ }// Defensive copy — json.RawMessage aliases the input buffer.p.pending = append(p.pending, append([]byte(nil), doc...))p.accumulated++@@ -138,3 +150,8 @@ func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {}return nil}++// full reports whether the window already holds every row it can take.+func (p *skipTakePipeline) full() bool {+ return p.take >= 0 && p.accumulated >= p.take+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.goindex 55bce60cb0..d0c5ae6ecd 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go@@ -30,13 +30,28 @@ func newTopPlan(n int) *planDoc {return p}-// driveTop feeds the given per-partition bodies through a TOP pipeline and-// returns the emitted rows.-func driveTop(t *testing.T, p *topPipeline, feed map[string][]byte) [][]byte {+func newOffsetLimitPlan(offset, limit int) *planDoc {+ p := &planDoc{}+ p.QueryInfo.Offset = intPtr(offset)+ p.QueryInfo.Limit = intPtr(limit)+ p.QueryInfo.RewrittenQuery = `SELECT c.name FROM c OFFSET 0 LIMIT 5`+ return p+}++func newSkipTake(t *testing.T, plan *planDoc, pkRangeIDs ...string) *skipTakePipeline {+ t.Helper()+ p, err := newSkipTakePipeline(plan, pkRangeIDs, `SELECT * FROM c`)+ require.NoError(t, err)+ return p+}++// driveTop feeds the given per-partition bodies through a skip/take pipeline+// and returns the emitted rows.+func driveTop(t *testing.T, p *skipTakePipeline, feed map[string][]byte) [][]byte {t.Helper()res, err := p.Run()require.NoError(t, err)- if p.n == 0 {+ if p.take == 0 {// Edge case: TOP 0 completes immediately without any partition work.assert.Empty(t, res.Requests)return nil@@ -62,46 +77,31 @@ func driveTop(t *testing.T, p *topPipeline, feed map[string][]byte) [][]byte {// ---------------------------------------------------------------- validation-func TestNewTopPipeline_RequiresNonNilTop(t *testing.T) {- plan := &planDoc{}- _, err := newTopPipeline(plan, []string{"0"})+func TestNewSkipTakePipeline_RequiresAWindow(t *testing.T) {+ _, err := newSkipTakePipeline(&planDoc{}, []string{"0"}, `SELECT * FROM c`)require.Error(t, err)- assert.Contains(t, err.Error(), "non-nil Top")+ assert.Contains(t, err.Error(), "requires TOP, OFFSET or LIMIT")}-func TestNewTopPipeline_RejectsNegativeN(t *testing.T) {- plan := newTopPlan(-1)- _, err := newTopPipeline(plan, []string{"0"})+func TestNewSkipTakePipeline_RejectsNegativeN(t *testing.T) {+ _, err := newSkipTakePipeline(newTopPlan(-1), []string{"0"}, `SELECT * FROM c`)require.Error(t, err)assert.Contains(t, err.Error(), "negative TOP")}-func TestNewTopPipeline_RejectsCompositionWithOrderBy(t *testing.T) {- plan := newTopPlan(5)- plan.QueryInfo.OrderBy = []string{"Ascending"}- _, err := newTopPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)-}--func TestNewTopPipeline_RejectsCompositionWithDistinct(t *testing.T) {- plan := newTopPlan(5)- plan.QueryInfo.DistinctType = "Unordered"- _, err := newTopPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)-}--func TestNewTopPipeline_RejectsCompositionWithAggregate(t *testing.T) {- plan := newTopPlan(5)- plan.QueryInfo.Aggregates = []string{"Count"}- _, err := newTopPipeline(plan, []string{"0"})- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)+// A bare TOP plan carries no rewrittenQuery, so the range answers the user's+// own query; an OFFSET/LIMIT plan carries one widened to cover the window.+func TestNewSkipTakePipeline_QueryComesFromThePlanWhenRewritten(t *testing.T) {+ assert.Equal(t, `SELECT * FROM c`, newSkipTake(t, newTopPlan(3), "0").Query())+ assert.Equal(t, `SELECT c.name FROM c OFFSET 0 LIMIT 5`,+ newSkipTake(t, newOffsetLimitPlan(2, 3), "0").Query())}// ---------------------------------------------------------------- semanticsfunc TestTopPipeline_SinglePartition_EmitsFirstN(t *testing.T) {plan := newTopPlan(3)- p, err := newTopPipeline(plan, []string{"0"})+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)require.NoError(t, err)rows := driveTop(t, p, map[string][]byte{@@ -114,7 +114,7 @@ func TestTopPipeline_SinglePartition_EmitsFirstN(t *testing.T) {func TestTopPipeline_FewerRowsThanN_EmitsAll(t *testing.T) {plan := newTopPlan(10)- p, err := newTopPipeline(plan, []string{"0"})+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)require.NoError(t, err)rows := driveTop(t, p, map[string][]byte{@@ -126,7 +126,7 @@ func TestTopPipeline_FewerRowsThanN_EmitsAll(t *testing.T) {func TestTopPipeline_MultiPartition_CapsAtN(t *testing.T) {plan := newTopPlan(4)- p, err := newTopPipeline(plan, []string{"A", "B", "C"})+ p, err := newSkipTakePipeline(plan, []string{"A", "B", "C"}, `SELECT * FROM c`)require.NoError(t, err)rows := driveTop(t, p, map[string][]byte{@@ -139,7 +139,7 @@ func TestTopPipeline_MultiPartition_CapsAtN(t *testing.T) {func TestTopPipeline_ZeroN_EmitsNothing(t *testing.T) {plan := newTopPlan(0)- p, err := newTopPipeline(plan, []string{"0"})+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)require.NoError(t, err)rows := driveTop(t, p, nil)@@ -149,7 +149,7 @@ func TestTopPipeline_ZeroN_EmitsNothing(t *testing.T) {func TestTopPipeline_EmptyPartitions(t *testing.T) {plan := newTopPlan(5)- p, err := newTopPipeline(plan, []string{"A", "B"})+ p, err := newSkipTakePipeline(plan, []string{"A", "B"}, `SELECT * FROM c`)require.NoError(t, err)rows := driveTop(t, p, map[string][]byte{@@ -162,8 +162,7 @@ func TestTopPipeline_EmptyPartitions(t *testing.T) {// A TOP larger than a page reads as many pages as it takes to collect N rows.func TestTopPipeline_ReadsPagesUntilNRowsAreInHand(t *testing.T) {- p, err := newTopPipeline(newTopPlan(5), []string{"0"})- require.NoError(t, err)+ p := newSkipTake(t, newTopPlan(5), "0")rows, read := drivePages(t, p, map[string][]page{"0": {@@ -179,8 +178,7 @@ func TestTopPipeline_ReadsPagesUntilNRowsAreInHand(t *testing.T) {// Once N rows are in hand the pipeline stops, however many pages the ranges// still hold.func TestTopPipeline_StopsAtNWithoutReadingTheNextPage(t *testing.T) {- p, err := newTopPipeline(newTopPlan(4), []string{"0"})- require.NoError(t, err)+ p := newSkipTake(t, newTopPlan(4), "0")rows, read := drivePages(t, p, map[string][]page{"0": {@@ -192,3 +190,40 @@ func TestTopPipeline_StopsAtNWithoutReadingTheNextPage(t *testing.T) {assert.Len(t, rows, 4)assert.Equal(t, map[string]int{"0": 2}, read)}++// ------------------------------------------------- OFFSET/LIMIT, no ORDER BY++// OFFSET/LIMIT without an ORDER BY is ordinary result paging. The gateway+// widens the per-partition query to cover the window, so every range offers+// at least as many rows as the window needs and the cut happens here.+func TestSkipTakePipeline_OffsetLimitCutsOneWindowGlobally(t *testing.T) {+ p := newSkipTake(t, newOffsetLimitPlan(2, 3), "0", "1")+ rows := driveTop(t, p, map[string][]byte{+ "0": mkTopBody(t, `{"x":1}`, `{"x":2}`, `{"x":3}`),+ "1": mkTopBody(t, `{"x":4}`, `{"x":5}`, `{"x":6}`),+ })+ assert.Equal(t, []string{`{"x":3}`, `{"x":4}`, `{"x":5}`}, stringsOf(rows))+}++func TestSkipTakePipeline_OffsetPastEveryRowEmitsNothing(t *testing.T) {+ p := newSkipTake(t, newOffsetLimitPlan(10, 3), "0")+ rows := driveTop(t, p, map[string][]byte{+ "0": mkTopBody(t, `{"x":1}`, `{"x":2}`),+ })+ assert.Empty(t, rows)+ assert.True(t, p.IsComplete())+}++// A window that straddles a page boundary has to keep reading the range.+func TestSkipTakePipeline_OffsetLimitReadsPagesUntilTheWindowIsFull(t *testing.T) {+ p := newSkipTake(t, newOffsetLimitPlan(3, 2), "0")+ rows, read := drivePages(t, p, map[string][]page{+ "0": {+ {body: mkTopBody(t, `{"x":1}`, `{"x":2}`), continuation: "0-page-2"},+ {body: mkTopBody(t, `{"x":3}`, `{"x":4}`), continuation: "0-page-3"},+ {body: mkTopBody(t, `{"x":5}`, `{"x":6}`)},+ },+ })+ assert.Equal(t, []string{`{"x":4}`, `{"x":5}`}, stringsOf(rows))+ assert.Equal(t, map[string]int{"0": 3}, read)+}