Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(cosmosdb): implement the query-plan operators the engine composed wrongly (#21895)

Danny Xu219bbdcbfc57
diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.go
index af21aa9fb9..2a14ad5f3a 100644
--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go
+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go
@@ -998,3 +998,280 @@ func TestCrossPartitionQueryLimit(t *testing.T) {
assert.Len(t, results[0].Rows, 7)
assert.EqualValues(t, 7, results[0].RowsCount)
}
+
+// ------------------------------------ composed operators, local emulator
+//
+// The query classes in BYT-10397: each composes two operators the gateway
+// reports in one plan, and the engine used to honour only one of them. Every
+// case here runs on the multi-page fixture, because two of them (TOP with
+// ORDER BY, and DISTINCT with a LIMIT) answer correctly on a single physical
+// partition however the engine is broken.
+
+// pagedQuery runs a query against the multi-page fixture and returns its rows
+// as raw JSON, in the order the engine emitted them.
+func pagedQuery(t *testing.T, container *azcosmos.ContainerClient, sql string) []string {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ items, err := QueryEveryPartition(ctx, container, sql, nil, 0)
+ require.NoError(t, err, "QueryEveryPartition: %s", sql)
+ rows := make([]string, 0, len(items))
+ for _, item := range items {
+ rows = append(rows, string(item))
+ }
+ return rows
+}
+
+// pagedBucketValues reads the bucket out of every row, for both spellings of
+// a projection: `{"bucket":"big-1"}` and the VALUE form's bare `"big-1"`.
+func pagedBucketValues(t *testing.T, rows []string) []string {
+ t.Helper()
+ out := make([]string, 0, len(rows))
+ for _, row := range rows {
+ var object struct {
+ Bucket *string `json:"bucket"`
+ }
+ if err := json.Unmarshal([]byte(row), &object); err == nil && object.Bucket != nil {
+ out = append(out, *object.Bucket)
+ continue
+ }
+ var scalar string
+ require.NoError(t, json.Unmarshal([]byte(row), &scalar), "row: %s", row)
+ out = append(out, scalar)
+ }
+ return out
+}
+
+// sortedPagedBuckets is the fixture's partition key values in ascending order.
+func sortedPagedBuckets() []string {
+ buckets := append(append([]string(nil), pagedBigBuckets...), pagedSmallBuckets...)
+ slices.Sort(buckets)
+ return buckets
+}
+
+// pagedBucketRows is how many rows each partition key value holds.
+func pagedBucketRows(bucket string) int {
+ if slices.Contains(pagedBigBuckets, bucket) {
+ return pagedBigRows
+ }
+ return pagedSmallRows
+}
+
+// TestCrossPartitionDistinctComposed covers DISTINCT alongside an ORDER BY or
+// a page. The service reports distinctType "Unordered" for the object form
+// even under ORDER BY, so the engine dropped the DISTINCT and returned every
+// row in the container; a paged DISTINCT ran the user's own query per range,
+// so each range applied the LIMIT to itself before anything was deduplicated.
+func TestCrossPartitionDistinctComposed(t *testing.T) {
+ container := pagedFixture(t)
+ buckets := sortedPagedBuckets()
+
+ t.Run("order_by_object_form", func(t *testing.T) {
+ assert.Equal(t, buckets, pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket`)))
+ })
+
+ t.Run("order_by_value_form", func(t *testing.T) {
+ assert.Equal(t, buckets, pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT VALUE c.bucket FROM c ORDER BY c.bucket`)))
+ })
+
+ t.Run("order_by_descending", func(t *testing.T) {
+ want := sortedPagedBuckets()
+ slices.Reverse(want)
+ assert.Equal(t, want, pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket DESC`)))
+ })
+
+ // An ordered DISTINCT pins the page against exact values; the unordered
+ // forms below can only be held to the count, since Cosmos defines no
+ // order for them.
+ t.Run("order_by_with_a_page", func(t *testing.T) {
+ assert.Equal(t, buckets[2:5], pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket OFFSET 2 LIMIT 3`)))
+ })
+
+ t.Run("limit_value_form", func(t *testing.T) {
+ got := pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT VALUE c.bucket FROM c OFFSET 0 LIMIT 3`))
+ assert.Len(t, got, 3)
+ assert.Subset(t, buckets, got)
+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(got))), 3, "values must be distinct")
+ })
+
+ t.Run("offset_limit_object_form", func(t *testing.T) {
+ got := pagedBucketValues(t,
+ pagedQuery(t, container, `SELECT DISTINCT c.bucket FROM c OFFSET 2 LIMIT 3`))
+ assert.Len(t, got, 3)
+ assert.Subset(t, buckets, got)
+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(got))), 3, "values must be distinct")
+ })
+}
+
+// TestCrossPartitionTopWithOrderBy covers TOP alongside an ORDER BY. The
+// rewritten per-range query carries the TOP, so the merge saw n rows from
+// every range and emitted all of them: n x partitions rows, in order.
+func TestCrossPartitionTopWithOrderBy(t *testing.T) {
+ container := pagedFixture(t)
+ rows := pagedRows()
+
+ t.Run("ascending", func(t *testing.T) {
+ assert.Equal(t, ascendingSeqs(0, 5),
+ pagedSeqs(t, container, `SELECT TOP 5 c.seq FROM c ORDER BY c.seq ASC`))
+ })
+
+ t.Run("descending", func(t *testing.T) {
+ want := ascendingSeqs(rows-5, rows)
+ slices.Reverse(want)
+ assert.Equal(t, want, pagedSeqs(t, container, `SELECT TOP 5 c.seq FROM c ORDER BY c.seq DESC`))
+ })
+
+ // A TOP past the gateway's page cap has to keep reading the ranges after
+ // the cap, and still stop at n.
+ t.Run("above_the_page_cap", func(t *testing.T) {
+ assert.Equal(t, ascendingSeqs(0, pagedPageSize+25),
+ pagedSeqs(t, container, fmt.Sprintf(
+ `SELECT TOP %d c.seq FROM c ORDER BY c.seq ASC`, pagedPageSize+25)))
+ })
+}
+
+// TestCrossPartitionGroupByOrder covers the order GROUP BY rows come back in.
+// The pipeline emitted them straight out of a Go map, so the same query
+// answered in a different order every run, and a GROUP BY the user ordered
+// reaches the engine as a plan with no order fields at all — the ORDER BY
+// survives only inside the rewritten query.
+func TestCrossPartitionGroupByOrder(t *testing.T) {
+ container := pagedFixture(t)
+
+ type group struct {
+ Bucket string `json:"bucket"`
+ N int `json:"n"`
+ }
+ groupsOf := func(t *testing.T, rows []string) []group {
+ t.Helper()
+ out := make([]group, 0, len(rows))
+ for _, row := range rows {
+ var g group
+ require.NoError(t, json.Unmarshal([]byte(row), &g), "row: %s", row)
+ out = append(out, g)
+ }
+ return out
+ }
+ want := make([]group, 0, len(sortedPagedBuckets()))
+ for _, bucket := range sortedPagedBuckets() {
+ want = append(want, group{Bucket: bucket, N: pagedBucketRows(bucket)})
+ }
+
+ t.Run("order_by_the_group_key", func(t *testing.T) {
+ assert.Equal(t, want, groupsOf(t, pagedQuery(t, container,
+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket`)))
+ })
+
+ t.Run("order_by_descending", func(t *testing.T) {
+ reversed := slices.Clone(want)
+ slices.Reverse(reversed)
+ assert.Equal(t, reversed, groupsOf(t, pagedQuery(t, container,
+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket DESC`)))
+ })
+
+ // Without an ORDER BY the order is Cosmos's to choose, but it must be the
+ // same on every run.
+ t.Run("unordered_is_stable_across_runs", func(t *testing.T) {
+ const sql = `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`
+ first := groupsOf(t, pagedQuery(t, container, sql))
+ assert.ElementsMatch(t, want, first)
+ for range 2 {
+ assert.Equal(t, first, groupsOf(t, pagedQuery(t, container, sql)))
+ }
+ })
+}
+
+// TestCrossPartitionEmptyAggregates covers an aggregate whose filter matches
+// no row. MIN and MAX failed the query outright, because an empty partition's
+// partial is an absent one; the aliased form answered `{"m":null}` where the
+// gateway omits the alias. COUNT and SUM really are 0 over an empty set and
+// are here as controls.
+func TestCrossPartitionEmptyAggregates(t *testing.T) {
+ container := pagedFixture(t)
+ const noRows = `WHERE c.seq > 99999999`
+
+ for _, tc := range []struct {
+ name string
+ sql string
+ want []string
+ }{
+ {"min_value", `SELECT VALUE MIN(c.seq) FROM c ` + noRows, nil},
+ {"max_value", `SELECT VALUE MAX(c.seq) FROM c ` + noRows, nil},
+ {"avg_value", `SELECT VALUE AVG(c.seq) FROM c ` + noRows, nil},
+ {"min_aliased", `SELECT MIN(c.seq) AS m FROM c ` + noRows, []string{`{}`}},
+ {"min_max_aliased", `SELECT MIN(c.seq) AS lo, MAX(c.seq) AS hi FROM c ` + noRows, []string{`{}`}},
+ {"count_value", `SELECT VALUE COUNT(1) FROM c ` + noRows, []string{`0`}},
+ {"count_aliased", `SELECT COUNT(1) AS n FROM c ` + noRows, []string{`{"n":0}`}},
+ {"sum_value", `SELECT VALUE SUM(c.seq) FROM c ` + noRows, []string{`0`}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got := pagedQuery(t, container, tc.sql)
+ require.Len(t, got, len(tc.want))
+ for i := range tc.want {
+ assert.JSONEq(t, tc.want[i], got[i])
+ }
+ })
+ }
+
+ // The same MIN and MAX over rows that do match: the VALUE form reads its
+ // partial from item2, which is where the count lives.
+ t.Run("min_max_over_rows_that_match", func(t *testing.T) {
+ assert.Equal(t, []string{`0`}, pagedQuery(t, container, `SELECT VALUE MIN(c.seq) FROM c`))
+ assert.Equal(t, []string{fmt.Sprint(pagedRows() - 1)},
+ pagedQuery(t, container, `SELECT VALUE MAX(c.seq) FROM c`))
+ })
+}
+
+// TestCrossPartitionOffsetLimitWithoutOrderBy covers ordinary result paging,
+// which the engine rejected although it advertises OffsetAndLimit. Cosmos
+// defines no order for it, so only the count and the rows' provenance hold.
+func TestCrossPartitionOffsetLimitWithoutOrderBy(t *testing.T) {
+ container := pagedFixture(t)
+
+ seqs := pagedSeqs(t, container, `SELECT c.seq FROM c OFFSET 2 LIMIT 3`)
+ require.Len(t, seqs, 3)
+ assert.Len(t, slices.Compact(slices.Sorted(slices.Values(seqs))), 3, "rows must not repeat")
+ for _, seq := range seqs {
+ assert.Less(t, seq, pagedRows())
+ assert.GreaterOrEqual(t, seq, 0)
+ }
+
+ // A window wider than the gateway's page cap reads several pages of a
+ // range before it is full.
+ wide := pagedSeqs(t, container, fmt.Sprintf(`SELECT c.seq FROM c OFFSET 0 LIMIT %d`, pagedPageSize+25))
+ assert.Len(t, wide, pagedPageSize+25)
+}
+
+// TestCrossPartitionAggregateWithOffsetLimit covers a page applied to an
+// aggregate. The query produces exactly one row, so OFFSET 1 skips past it and
+// the gateway answers with nothing; the engine returned the row regardless.
+func TestCrossPartitionAggregateWithOffsetLimit(t *testing.T) {
+ container := pagedFixture(t)
+
+ assert.Empty(t, pagedQuery(t, container, `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`))
+ assert.Equal(t, []string{fmt.Sprint(pagedRows())},
+ pagedQuery(t, container, `SELECT VALUE COUNT(1) FROM c OFFSET 0 LIMIT 1`))
+ assert.Empty(t, pagedQuery(t, container, `SELECT COUNT(1) AS n FROM c OFFSET 1 LIMIT 1`))
+}
+
+// TestCrossPartitionDistinctWithOrderByThroughTheDriver runs the worst of
+// these through QueryConn, the entry point the SQL editor uses, so the SQL
+// validator is in the path too.
+func TestCrossPartitionDistinctWithOrderByThroughTheDriver(t *testing.T) {
+ driver := emulatorFixture(t)
+ rows := runQuery(t, driver, crossPartitionContainer, `SELECT DISTINCT c.country FROM c ORDER BY c.country`)
+ got := make([]string, 0, len(rows))
+ for _, row := range rows {
+ var object struct {
+ Country string `json:"country"`
+ }
+ require.NoError(t, json.Unmarshal(row, &object))
+ got = append(got, object.Country)
+ }
+ assert.Equal(t, slices.Sorted(slices.Values(crossPartitionCountries)), got)
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go
index 2eb8e0ea63..044b23aac3 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg.go
@@ -2,6 +2,7 @@ package gonative
import (
"encoding/json"
+ "fmt"
"strconv"
"github.com/pkg/errors"
@@ -19,8 +20,11 @@ import (
// Max — {"max": <value>, "count": <int>}
//
// finalize returns the final scalar as a json.RawMessage so the pipeline can
-// emit it without re-marshaling. Returns an empty value (JSON null) when no
-// contributing rows were observed (e.g. Avg over an empty set).
+// emit it without re-marshaling. It returns a nil message when the aggregate
+// has no value: Cosmos reports MIN, MAX and AVG over an empty set as
+// undefined, which the gateway renders as no row at all in the VALUE form and
+// as an absent alias in the aliased form. COUNT and SUM over an empty set are
+// 0, and the gateway sends that 0 as a partial, so they never finalize nil.
type aggregator interface {
feedPartial(raw json.RawMessage) error
finalize() (json.RawMessage, error)
@@ -47,6 +51,45 @@ func newAggregator(kind string) aggregator {
return nil
}
+// aggregatePartial returns the partial feedPartial expects for one aggregate,
+// given the gateway's {item, item2} document entry. MIN and MAX read item2:
+// only it reports how many rows the partition contributed, which is how a
+// partition that matched nothing contributes nothing. item2 is absent where
+// the gateway short-circuited to a single-partition shape, and item is then
+// the value itself. Every other kind carries its partial in item alone.
+//
+// Returns nil when the gateway left the partial out, which it does for an
+// aggregate over a partition that matched no row.
+func aggregatePartial(kind string, it aggItem) json.RawMessage {
+ switch kind {
+ case "Min", "Max":
+ if len(it.Item2) > 0 {
+ return it.Item2
+ }
+ if len(it.Item) == 0 {
+ return nil
+ }
+ key := "min"
+ if kind == "Max" {
+ key = "max"
+ }
+ return fmt.Appendf(nil, `{%q:%s,"count":1}`, key, it.Item)
+ default:
+ return it.Item
+ }
+}
+
+// feedAggregatePartial folds one partition document's partial into a. A
+// partial the gateway left out is not an error: the partition had no row to
+// contribute.
+func feedAggregatePartial(a aggregator, kind string, it aggItem) error {
+ partial := aggregatePartial(kind, it)
+ if len(partial) == 0 {
+ return nil
+ }
+ return a.feedPartial(partial)
+}
+
// -----------------------------------------------------------------------------
// Count
@@ -87,7 +130,7 @@ func (a *sumAgg) feedPartial(raw json.RawMessage) error {
func (a *sumAgg) finalize() (json.RawMessage, error) {
if !a.seen {
- return json.RawMessage(`null`), nil
+ return nil, nil
}
return numberRawMessage(a.total), nil
}
@@ -129,7 +172,7 @@ func (a *minAgg) feedPartial(raw json.RawMessage) error {
func (a *minAgg) finalize() (json.RawMessage, error) {
if a.best == nil {
- return json.RawMessage(`null`), nil
+ return nil, nil
}
return a.best, nil
}
@@ -170,7 +213,7 @@ func (a *maxAgg) feedPartial(raw json.RawMessage) error {
func (a *maxAgg) finalize() (json.RawMessage, error) {
if a.best == nil {
- return json.RawMessage(`null`), nil
+ return nil, nil
}
return a.best, nil
}
@@ -200,9 +243,7 @@ func (a *avgAgg) feedPartial(raw json.RawMessage) error {
func (a *avgAgg) finalize() (json.RawMessage, error) {
if a.count == 0 {
- // Matches Cosmos semantics: AVG over an empty set is undefined;
- // gateway serializes this as JSON null for cross-partition results.
- return json.RawMessage(`null`), nil
+ return nil, nil
}
return numberRawMessage(a.sum / float64(a.count)), nil
}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
index 526f2691b6..a08a7ddfdb 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
@@ -51,9 +51,9 @@ func TestCountAgg_SumsAcrossPartitions(t *testing.T) {
// ---------------------------------------------------------- Sum
-func TestSumAgg_EmptyFinalizesToNull(t *testing.T) {
+func TestSumAgg_EmptyFinalizesToUndefined(t *testing.T) {
out := feedFinal(t, newAggregator("Sum"))
- assert.Equal(t, json.RawMessage(`null`), out)
+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")
}
func TestSumAgg_SumsAcrossPartitions(t *testing.T) {
@@ -64,9 +64,9 @@ func TestSumAgg_SumsAcrossPartitions(t *testing.T) {
// ---------------------------------------------------------- Avg
-func TestAvgAgg_EmptyFinalizesToNull(t *testing.T) {
+func TestAvgAgg_EmptyFinalizesToUndefined(t *testing.T) {
out := feedFinal(t, newAggregator("Avg"))
- assert.Equal(t, json.RawMessage(`null`), out)
+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")
}
func TestAvgAgg_SinglePartition(t *testing.T) {
@@ -88,11 +88,43 @@ func TestAvgAgg_ZeroCountPartialContributesNothing(t *testing.T) {
assert.Equal(t, json.RawMessage(`2`), out)
}
+// ---------------------------------------------------------- partial routing
+
+// A MIN or MAX partial comes from item2, the only one that reports how many
+// rows the partition contributed; a partition that matched none contributes
+// nothing at all.
+func TestAggregatePartial_MinMaxPreferItem2(t *testing.T) {
+ both := aggItem{Item: json.RawMessage(`7`), Item2: json.RawMessage(`{"min":7,"count":3}`)}
+ assert.JSONEq(t, `{"min":7,"count":3}`, string(aggregatePartial("Min", both)))
+
+ onlyItem := aggItem{Item: json.RawMessage(`7`)}
+ assert.JSONEq(t, `{"min":7,"count":1}`, string(aggregatePartial("Min", onlyItem)))
+ assert.JSONEq(t, `{"max":7,"count":1}`, string(aggregatePartial("Max", onlyItem)))
+
+ assert.Nil(t, aggregatePartial("Min", aggItem{}))
+ assert.Equal(t, json.RawMessage(`7`), aggregatePartial("Count", onlyItem))
+}
+
+// An aggregate over a partition that matched no row has no partial for the
+// gateway to send, and the aggregate still has to finalize.
+func TestFeedAggregatePartial_SkipsAnAbsentPartial(t *testing.T) {
+ a := newAggregator("Min")
+ require.NoError(t, feedAggregatePartial(a, "Min", aggItem{}))
+ out, err := a.finalize()
+ require.NoError(t, err)
+ assert.Nil(t, out)
+
+ require.NoError(t, feedAggregatePartial(a, "Min", aggItem{Item2: json.RawMessage(`{"count":0}`)}))
+ out, err = a.finalize()
+ require.NoError(t, err)
+ assert.Nil(t, out)
+}
+
// ---------------------------------------------------------- Min
-func TestMinAgg_EmptyFinalizesToNull(t *testing.T) {
+func TestMinAgg_EmptyFinalizesToUndefined(t *testing.T) {
out := feedFinal(t, newAggregator("Min"))
- assert.Equal(t, json.RawMessage(`null`), out)
+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")
}
func TestMinAgg_SinglePartitionNumeric(t *testing.T) {
@@ -128,9 +160,9 @@ func TestMinAgg_MixedTypesUsesCosmosOrdering(t *testing.T) {
// ---------------------------------------------------------- Max
-func TestMaxAgg_EmptyFinalizesToNull(t *testing.T) {
+func TestMaxAgg_EmptyFinalizesToUndefined(t *testing.T) {
out := feedFinal(t, newAggregator("Max"))
- assert.Equal(t, json.RawMessage(`null`), out)
+ assert.Nil(t, out, "no contributing row means no value, which the pipelines render as no row or an absent alias")
}
func TestMaxAgg_AcrossPartitions_TakesLargest(t *testing.T) {
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go
index 330eb408ae..397d34983e 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct.go
@@ -57,6 +57,15 @@ type distinctPipeline struct {
fanout rangeFanout
closed bool
+ // offset and limit are the plan's global skip and take, applied after
+ // dedup: the gateway strips OFFSET/LIMIT and TOP from the rewritten query
+ // so that each range reports its own distinct values in full and the page
+ // is cut once, over the whole result. A negative limit means unlimited.
+ offset int
+ limit int
+ skipped int
+ taken int
+
// Pending items not yet returned to the caller. Populated during
// ProvideData as each partition's response is de-duplicated against the
// seen set.
@@ -68,19 +77,30 @@ type distinctPipeline struct {
seen map[string]struct{}
}
-// newDistinctPipeline builds a pipeline for a plan whose distinctType is
-// "Unordered". Returns an error if the plan's distinctType is "Ordered" —
-// that form composes with Stage 3's ORDER BY merge and is not handled here.
-func newDistinctPipeline(plan *planDoc, pkRangeIDs []string) (*distinctPipeline, error) {
- if plan.QueryInfo.DistinctType != "Unordered" {
- // "Ordered" is Stage 3's responsibility; anything else is unexpected.
+// newDistinctPipeline builds a pipeline for a DISTINCT plan. "Ordered" and
+// "Unordered" differ only in whether the gateway promises sorted input, which
+// a global seen-set does not need; any other distinctType is a shape the
+// engine has not seen and rejects.
+func newDistinctPipeline(plan *planDoc, pkRangeIDs []string, query string) (*distinctPipeline, error) {
+ if plan.QueryInfo.DistinctType != "Unordered" && plan.QueryInfo.DistinctType != "Ordered" {
return nil, ErrUnsupportedPlanFeature
}
- // DISTINCT plans carry an empty rewrittenQuery — run the original query
- // against each partition. The pipeline needs the original query text to
- // populate QueryRequest.Query; the caller passes it via CreateQueryPipeline.
+ offset, limit, err := planSkipTake(plan.QueryInfo)
+ if err != nil {
+ return nil, err
+ }
+ // A plain DISTINCT plan carries no rewrittenQuery, so each range answers
+ // the user's own query. A DISTINCT the user paged or capped does carry
+ // one — the same DISTINCT with the OFFSET/LIMIT or TOP taken off — and
+ // running that instead is what lets this pipeline cut the page globally.
+ if plan.QueryInfo.RewrittenQuery != "" {
+ query = plan.QueryInfo.RewrittenQuery
+ }
return &distinctPipeline{
+ query: query,
fanout: newRangeFanout(pkRangeIDs),
+ offset: offset,
+ limit: limit,
seen: make(map[string]struct{}),
}, nil
}
@@ -95,6 +115,11 @@ func (p *distinctPipeline) IsComplete() bool {
if len(p.pending) > 0 {
return false
}
+ if p.limit >= 0 && p.taken >= p.limit {
+ // LIMIT satisfied — whatever the ranges still hold cannot change the
+ // answer, so the pipeline stops asking for it.
+ return true
+ }
return p.fanout.allDrained()
}
@@ -116,6 +141,9 @@ func (p *distinctPipeline) Run() (*queryengine.PipelineResult, error) {
IsCompleted: p.IsComplete(),
}, nil
}
+ if p.IsComplete() {
+ return &queryengine.PipelineResult{IsCompleted: true}, nil
+ }
if reqs := p.fanout.requests(p.query); len(reqs) > 0 {
return &queryengine.PipelineResult{Requests: reqs}, nil
}
@@ -124,7 +152,8 @@ func (p *distinctPipeline) Run() (*queryengine.PipelineResult, error) {
// ProvideData implements queryengine.QueryPipeline. Each incoming partition
// response is parsed and its documents are fed through the global seen set.
-// First-sightings become pending items; duplicates are silently dropped.
+// First-sightings pass to OFFSET/LIMIT and then become pending items;
+// duplicates are silently dropped.
func (p *distinctPipeline) ProvideData(results []queryengine.QueryResult) error {
for _, r := range results {
resp, err := parseDistinctPartitionResponse(r.Data)
@@ -141,11 +170,19 @@ func (p *distinctPipeline) ProvideData(results []queryengine.QueryResult) error
continue
}
p.seen[key] = struct{}{}
+ if p.skipped < p.offset {
+ p.skipped++
+ continue
+ }
+ if p.limit >= 0 && p.taken >= p.limit {
+ break
+ }
// Preserve the partition's original document bytes in the output —
// callers receive what the gateway returned, not our canonicalized
// form. Copy defensively: json.RawMessage is aliased to the input
// buffer, which the SDK may reuse for later pages.
p.pending = append(p.pending, append([]byte(nil), doc...))
+ p.taken++
}
if err := p.fanout.note(r); err != nil {
return err
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go
index 3e0be9e63f..fd55f7613f 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/distinct_test.go
@@ -88,12 +88,15 @@ func rawMsgs(ss ...string) []json.RawMessage {
return out
}
+// distinctTestQuery stands in for the user's query, which a DISTINCT plan
+// with no rewrittenQuery sends to every range unchanged.
+const distinctTestQuery = `SELECT DISTINCT c.country FROM c`
+
func TestDistinctPipeline_SinglePartitionEmitsAll(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
- p.query = `SELECT DISTINCT c.country FROM c`
items := feedSingle(t, p, rawMsgs(`{"country":"AD"}`, `{"country":"AE"}`, `{"country":"DE"}`))
assert.Len(t, items, 3)
@@ -103,7 +106,7 @@ func TestDistinctPipeline_SinglePartitionEmitsAll(t *testing.T) {
func TestDistinctPipeline_DedupesWithinPartition(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
items := feedSingle(t, p, rawMsgs(`{"country":"AE"}`, `{"country":"AE"}`, `{"country":"AD"}`))
@@ -113,7 +116,7 @@ func TestDistinctPipeline_DedupesWithinPartition(t *testing.T) {
func TestDistinctPipeline_DedupesAcrossPartitions(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0", "1", "2"})
+ p, err := newDistinctPipeline(plan, []string{"0", "1", "2"}, distinctTestQuery)
require.NoError(t, err)
// Kick off the request fan-out.
@@ -144,7 +147,7 @@ func TestDistinctPipeline_DedupesAcrossPartitions(t *testing.T) {
func TestDistinctPipeline_TreatsKeyReorderedObjectsAsDuplicates(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
items := feedSingle(t, p, rawMsgs(
@@ -158,7 +161,7 @@ func TestDistinctPipeline_DistinctValueScalars(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
plan.QueryInfo.HasSelectValue = true
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
// DISTINCT VALUE emits bare scalars in Documents[].
@@ -166,10 +169,10 @@ func TestDistinctPipeline_DistinctValueScalars(t *testing.T) {
assert.Len(t, items, 3)
}
-func TestDistinctPipeline_RejectsOrderedDistinctType(t *testing.T) {
+func TestDistinctPipeline_RejectsUnknownDistinctType(t *testing.T) {
plan := &planDoc{}
- plan.QueryInfo.DistinctType = "Ordered"
- _, err := newDistinctPipeline(plan, []string{"0"})
+ plan.QueryInfo.DistinctType = "SomethingNew"
+ _, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
}
@@ -178,7 +181,7 @@ func TestDistinctPipeline_RejectsOrderedDistinctType(t *testing.T) {
func TestDistinctPipeline_DedupesAcrossPages(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
rows, read := drivePages(t, p, map[string][]page{
@@ -208,7 +211,7 @@ func mkDistinctBody(t *testing.T, values ...string) []byte {
func TestDistinctPipeline_HighCardinality(t *testing.T) {
plan := &planDoc{}
plan.QueryInfo.DistinctType = "Unordered"
- p, err := newDistinctPipeline(plan, []string{"0"})
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
require.NoError(t, err)
_, _ = p.Run()
@@ -221,3 +224,86 @@ func TestDistinctPipeline_HighCardinality(t *testing.T) {
{PartitionKeyRangeID: "0", Data: body},
}))
}
+
+// ------------------------------------------------- OFFSET, LIMIT and TOP
+
+// A DISTINCT the user paged carries a rewrittenQuery with the OFFSET/LIMIT
+// taken off, so every range reports its own distinct values in full and the
+// page is cut once, here. Sending the user's query instead would make each
+// range apply the LIMIT to itself.
+func TestNewDistinctPipeline_PagedPlanRunsTheRewrittenQuery(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Limit = intPtr(3)
+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`
+ p, err := newDistinctPipeline(plan, []string{"0"}, `SELECT DISTINCT c.country FROM c OFFSET 0 LIMIT 3`)
+ require.NoError(t, err)
+ assert.Equal(t, `SELECT DISTINCT c.country FROM c`, p.Query())
+}
+
+func TestDistinctPipeline_LimitAppliesAfterDedup(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Limit = intPtr(2)
+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
+ require.NoError(t, err)
+
+ items := feedSingle(t, p, rawMsgs(`"AE"`, `"AE"`, `"BR"`, `"CN"`))
+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(items))
+}
+
+func TestDistinctPipeline_OffsetSkipsDistinctValuesNotRows(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Offset = intPtr(1)
+ plan.QueryInfo.Limit = intPtr(2)
+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
+ require.NoError(t, err)
+
+ items := feedSingle(t, p, rawMsgs(`"AE"`, `"AE"`, `"BR"`, `"CN"`, `"DE"`))
+ assert.Equal(t, []string{`"BR"`, `"CN"`}, stringsOf(items))
+}
+
+// TOP on a DISTINCT reaches the engine as a plan field exactly like LIMIT,
+// and caps the same way.
+func TestDistinctPipeline_TopCapsTheDedupedStream(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Top = intPtr(2)
+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
+ require.NoError(t, err)
+
+ items := feedSingle(t, p, rawMsgs(`"AE"`, `"BR"`, `"CN"`))
+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(items))
+}
+
+// Once the LIMIT is met the pipeline must stop, however many pages the ranges
+// still hold: a satisfied page costs request charge for rows nobody sees.
+func TestDistinctPipeline_StopsAtTheLimitWithoutReadingTheNextPage(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Limit = intPtr(2)
+ plan.QueryInfo.RewrittenQuery = `SELECT DISTINCT c.country FROM c`
+ p, err := newDistinctPipeline(plan, []string{"0"}, distinctTestQuery)
+ require.NoError(t, err)
+
+ rows, read := drivePages(t, p, map[string][]page{
+ "0": {
+ {body: mkDistinctBody(t, `"AE"`, `"BR"`), continuation: "0-page-2"},
+ {body: mkDistinctBody(t, `"CN"`, `"DE"`)},
+ },
+ })
+ assert.Equal(t, []string{`"AE"`, `"BR"`}, stringsOf(rows))
+ assert.Equal(t, map[string]int{"0": 1}, read)
+}
+
+func stringsOf(items [][]byte) []string {
+ out := make([]string, 0, len(items))
+ for _, it := range items {
+ out = append(out, string(it))
+ }
+ return out
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go b/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go
index 8dd9f06aa7..082715567d 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/engine.go
@@ -1,38 +1,38 @@
package gonative
import (
+ "reflect"
+ "slices"
+
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
+ "github.com/pkg/errors"
)
-// Engine is the pure-Go implementation of queryengine.QueryEngine.
-// It is constructed by Default(); callers should not instantiate Engine
-// directly because future stages may add required initialization.
-type Engine struct {
- // (future stages add fields: supportedFeatures set, cardinality caps, etc.)
-}
+// Engine is the pure-Go implementation of queryengine.QueryEngine. Construct
+// it with Default() and treat the value as opaque.
+type Engine struct{}
// Compile-time check that *Engine satisfies the interface.
var _ queryengine.QueryEngine = (*Engine)(nil)
-// Default returns a new Engine with Stage-0 capabilities (none).
-// Callers should treat the returned value as opaque.
+// Default returns a new Engine.
func Default() *Engine {
return &Engine{}
}
// supportedFeatures enumerates the gateway-known feature names the engine
// can serve. The string is comma-separated per the gateway's query-plan
-// protocol; new feature names are appended as each stage lands.
+// protocol.
//
-// Aggregate — VALUE-form aggregate queries (Stage 1)
-// NonValueAggregate — aliased-form aggregates (Stage 1)
-// MultipleAggregates — multi-aggregate in a single SELECT (Stage 1)
-// DistinctValue — SELECT DISTINCT VALUE c.field FROM c (Stage 2)
-// Distinct — SELECT DISTINCT c.field FROM c — object form (Stage 2)
-// OrderBy — single-key ORDER BY (Stage 3)
-// Top — SELECT TOP N FROM c (Stage 4)
-// OffsetAndLimit — SELECT … ORDER BY … OFFSET N LIMIT M (Stage 5)
-// GroupBy — SELECT … FROM c GROUP BY c.field (Stage 6)
+// Aggregate — VALUE-form aggregate queries
+// NonValueAggregate — aliased-form aggregates
+// MultipleAggregates — multi-aggregate in a single SELECT
+// DistinctValue — SELECT DISTINCT VALUE c.field FROM c
+// Distinct — SELECT DISTINCT c.field FROM c — object form
+// OrderBy — single-key ORDER BY
+// Top — SELECT TOP N FROM c
+// OffsetAndLimit — OFFSET N LIMIT M
+// GroupBy — SELECT … FROM c GROUP BY c.field
const supportedFeatures = "Aggregate,NonValueAggregate,MultipleAggregates,Distinct,DistinctValue,OrderBy,Top,OffsetAndLimit,GroupBy"
// SupportedFeatures implements queryengine.QueryEngine.
@@ -40,85 +40,117 @@ func (*Engine) SupportedFeatures() string {
return supportedFeatures
}
+// Names of the planQueryInfo fields a dispatch branch claims. A plan field the
+// chosen pipeline does not claim fails the query — see requireHandled.
+const (
+ fieldRewrittenQuery = "RewrittenQuery"
+ fieldHasSelectValue = "HasSelectValue"
+ fieldAggregates = "Aggregates"
+ fieldGroupByAliases = "GroupByAliases"
+ fieldGroupByAliasToAggregateType = "GroupByAliasToAggregateType"
+ fieldDistinctType = "DistinctType"
+ fieldTop = "Top"
+ fieldOffset = "Offset"
+ fieldLimit = "Limit"
+ fieldOrderBy = "OrderBy"
+ fieldOrderByExpressions = "OrderByExpressions"
+ fieldGroupByExpressions = "GroupByExpressions"
+)
+
// CreateQueryPipeline implements queryengine.QueryEngine. It parses the plan,
-// rejects any feature the engine does not yet support, then dispatches to the
-// appropriate pipeline implementation.
+// picks the pipeline for the plan's shape and rejects any plan field that
+// pipeline cannot honour.
func (*Engine) CreateQueryPipeline(query string, plan string, pkranges string) (queryengine.QueryPipeline, error) {
p, err := parsePlan([]byte(plan))
if err != nil {
return nil, err
}
- if err := rejectUnsupportedPlan(p); err != nil {
- return nil, err
- }
rangeIDs, err := parsePKRangeIDs([]byte(pkranges))
if err != nil {
return nil, err
}
+ qi := p.QueryInfo
switch {
- case len(p.QueryInfo.OrderBy) > 0:
+ case len(qi.OrderBy) > 0:
+ // ORDER BY owns every operator that applies to an ordered stream:
+ // DISTINCT dedups it, TOP and OFFSET/LIMIT cut it.
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldOrderBy, fieldOrderByExpressions,
+ fieldDistinctType, fieldTop, fieldOffset, fieldLimit); err != nil {
+ return nil, err
+ }
return newOrderByPipeline(p, rangeIDs)
- case p.QueryInfo.DistinctType == "Unordered":
- pipe, err := newDistinctPipeline(p, rangeIDs)
- if err != nil {
+ case qi.DistinctType != "":
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldDistinctType, fieldTop, fieldOffset, fieldLimit); err != nil {
return nil, err
}
- pipe.query = query
- return pipe, nil
- case len(p.QueryInfo.Aggregates) > 0:
- pipe, err := newValuePipeline(p, rangeIDs)
- if err != nil {
+ return newDistinctPipeline(p, rangeIDs, query)
+ case len(qi.Aggregates) > 0:
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldAggregates, fieldOffset, fieldLimit); err != nil {
return nil, err
}
- return pipe, nil
- case len(p.QueryInfo.GroupByExpressions) > 0:
+ return newValuePipeline(p, rangeIDs)
+ case len(qi.GroupByExpressions) > 0:
// GROUP BY plans carry groupByExpressions; the aliased-aggregate
- // (single-group) path used by Stage 1 has groupByAliasToAggregateType
- // populated but groupByExpressions empty. The check order matters.
+ // (single-group) path has groupByAliasToAggregateType populated but
+ // groupByExpressions empty. The check order matters.
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldGroupByExpressions, fieldGroupByAliases, fieldGroupByAliasToAggregateType); err != nil {
+ return nil, err
+ }
return newGroupPipeline(p, rangeIDs)
- case len(p.QueryInfo.GroupByAliasToAggregateType) > 0:
- pipe, err := newAliasedPipeline(p, rangeIDs)
- if err != nil {
+ case len(qi.GroupByAliasToAggregateType) > 0:
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldGroupByAliases, fieldGroupByAliasToAggregateType, fieldOffset, fieldLimit); err != nil {
return nil, err
}
- return pipe, nil
- case p.QueryInfo.Top != nil:
- pipe, err := newTopPipeline(p, rangeIDs)
- if err != nil {
+ return newAliasedPipeline(p, rangeIDs)
+ case qi.Top != nil || qi.Offset != nil || qi.Limit != nil:
+ if err := requireHandled(qi, fieldRewrittenQuery, fieldHasSelectValue,
+ fieldTop, fieldOffset, fieldLimit); err != nil {
return nil, err
}
- pipe.query = query
- return pipe, nil
+ return newSkipTakePipeline(p, rangeIDs, query)
default:
- return nil, ErrUnsupportedPlanFeature
+ return nil, errors.Wrap(ErrUnsupportedPlanFeature, "no pipeline for this plan shape")
}
}
-// rejectUnsupportedPlan returns ErrUnsupportedPlanFeature when the plan uses
-// any feature beyond the engine's current scope. Later stages will prune
-// entries from this guard as their operators come online.
-func rejectUnsupportedPlan(p *planDoc) error {
- qi := p.QueryInfo
- // "Ordered" DISTINCT composes with ORDER BY merge; reject until Stage 3
- // wires composition through the merge heap. Stage 3 itself handles plain
- // ORDER BY — see the dispatch above.
- if qi.DistinctType == "Ordered" {
- return ErrUnsupportedPlanFeature
- }
- if qi.DistinctType != "" && qi.DistinctType != "None" && qi.DistinctType != "Unordered" {
- return ErrUnsupportedPlanFeature
- }
- // OFFSET/LIMIT always composes with ORDER BY per Cosmos syntax; when that
- // composition lands the plan has both qi.OrderBy and qi.Offset/qi.Limit,
- // and the dispatch routes to newOrderByPipeline which handles them.
- // Plans that set offset/limit without an ORDER BY fall through to the
- // default case in CreateQueryPipeline and are rejected there.
- if qi.HasNonStreamingOrderBy {
- return ErrUnsupportedPlanFeature
+// requireHandled rejects a plan whose queryInfo sets a field the chosen
+// pipeline does not claim. It walks planQueryInfo by reflection rather than
+// checking a fixed list, so a field added to the struct later is rejected
+// until a pipeline claims it: an operator the pipeline cannot apply must fail
+// the query rather than vanish from the answer.
+func requireHandled(qi planQueryInfo, handled ...string) error {
+ v := reflect.ValueOf(qi)
+ t := v.Type()
+ for i := range t.NumField() {
+ name := t.Field(i).Name
+ if !planFieldSet(v.Field(i)) || slices.Contains(handled, name) {
+ continue
+ }
+ return errors.Wrapf(ErrUnsupportedPlanFeature, "pipeline cannot honour queryInfo.%s", name)
}
return nil
}
+// planFieldSet reports whether a queryInfo field carries an instruction. A
+// plan that spells a field out as empty — `"aggregates": []` — says nothing,
+// and reflect.Value.IsZero would call that slice set because it is non-nil.
+// A pointer field is set whenever it is non-nil: `"top": 0` means TOP 0.
+func planFieldSet(v reflect.Value) bool {
+ switch v.Kind() {
+ case reflect.Slice, reflect.Map, reflect.String:
+ return v.Len() > 0
+ case reflect.Pointer:
+ return !v.IsNil()
+ default:
+ return !v.IsZero()
+ }
+}
+
// CreateReadManyPipeline implements queryengine.QueryEngine.
func (*Engine) CreateReadManyPipeline(_ []queryengine.ItemIdentity, _ string, _ string, _ uint8, _ []string) (queryengine.QueryPipeline, error) {
return nil, ErrUnsupportedPlanFeature
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go
index 8bf2d215f9..08eb6f5694 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/engine_test.go
@@ -1,31 +1,28 @@
-package gonative_test
+package gonative
import (
- "errors"
+ "os"
+ "path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/plugin/db/cosmosdb/queryengine/gonative"
)
func TestDefaultReturnsNonNilEngine(t *testing.T) {
- e := gonative.Default()
+ e := Default()
require.NotNil(t, e)
}
-func TestDefaultAdvertisesStage1Features(t *testing.T) {
- // Stage 1 advertises aggregate-family features so the gateway emits plans
- // for VALUE + aliased + multi-aggregate queries.
- feats := gonative.Default().SupportedFeatures()
+func TestDefaultAdvertisesAggregateFeatures(t *testing.T) {
+ feats := Default().SupportedFeatures()
assert.Contains(t, feats, "Aggregate")
assert.Contains(t, feats, "NonValueAggregate")
assert.Contains(t, feats, "MultipleAggregates")
}
func TestDefaultCreateQueryPipelineRejectsUnparseablePlan(t *testing.T) {
- _, err := gonative.Default().CreateQueryPipeline("SELECT * FROM c", "not json", "{}")
+ _, err := Default().CreateQueryPipeline("SELECT * FROM c", "not json", "{}")
require.Error(t, err)
}
@@ -34,20 +31,115 @@ func TestDefaultCreateQueryPipelineRejectsUnsupportedPlan(t *testing.T) {
// is out of the engine's scope — the design spec explicitly excludes it.
plan := `{"partitionedQueryExecutionInfoVersion":2,"queryInfo":{"hasNonStreamingOrderBy":true}}`
pkranges := `{"PartitionKeyRanges":[{"id":"0"}]}`
- _, err := gonative.Default().CreateQueryPipeline("SELECT ...", plan, pkranges)
- require.Error(t, err)
- assert.True(t, errors.Is(err, gonative.ErrUnsupportedPlanFeature))
+ _, err := Default().CreateQueryPipeline("SELECT ...", plan, pkranges)
+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
}
func TestDefaultCreateReadManyPipelineReturnsUnsupported(t *testing.T) {
- _, err := gonative.Default().CreateReadManyPipeline(nil, "{}", "", 0, nil)
- require.Error(t, err)
- assert.True(t, errors.Is(err, gonative.ErrUnsupportedPlanFeature))
+ _, err := Default().CreateReadManyPipeline(nil, "{}", "", 0, nil)
+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
}
func TestDefaultReturnsDistinctEngineInstances(t *testing.T) {
// Each caller gets its own instance — enables per-engine caches in later stages.
- a := gonative.Default()
- b := gonative.Default()
+ a := Default()
+ b := Default()
assert.NotSame(t, a, b)
}
+
+// recordedPlan loads a query plan the gateway really emitted for one of the
+// BYT-10397 query shapes, as the engine receives it.
+func recordedPlan(t *testing.T, name string) string {
+ t.Helper()
+ raw, err := os.ReadFile(filepath.Join("testdata", "query_"+name+"_plan.json"))
+ require.NoError(t, err)
+ return string(raw)
+}
+
+// TestCreateQueryPipelineRejectsUnhandledPlanField drives the dispatch with
+// plans whose composition no pipeline implements. The query has to fail rather
+// than come back missing an operator, which is what every wrong row count in
+// BYT-10397 was.
+func TestCreateQueryPipelineRejectsUnhandledPlanField(t *testing.T) {
+ const pkranges = `{"PartitionKeyRanges":[{"id":"0"},{"id":"1"}]}`
+ for _, tc := range []struct {
+ plan string
+ query string
+ field string
+ }{
+ {"groupby_top", `SELECT TOP 3 c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`, "Top"},
+ {"groupby_offset_limit", `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket OFFSET 1 LIMIT 2`, "Offset"},
+ } {
+ t.Run(tc.plan, func(t *testing.T) {
+ _, err := Default().CreateQueryPipeline(tc.query, recordedPlan(t, tc.plan), pkranges)
+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+ assert.Contains(t, err.Error(), "queryInfo."+tc.field)
+ })
+ }
+}
+
+// TestCreateQueryPipelineAcceptsRecordedCompositions pins the dispatch against
+// the plan shapes BYT-10397 collected: each composes two operators, and the
+// pipeline the dispatch picks has to claim both of them.
+func TestCreateQueryPipelineAcceptsRecordedCompositions(t *testing.T) {
+ const pkranges = `{"PartitionKeyRanges":[{"id":"0"},{"id":"1"}]}`
+ for _, tc := range []struct{ plan, query string }{
+ {"distinct_orderby", `SELECT DISTINCT c.bucket FROM c ORDER BY c.bucket`},
+ {"distinct_orderby_value", `SELECT DISTINCT VALUE c.bucket FROM c ORDER BY c.bucket`},
+ {"top_orderby_asc", `SELECT TOP 5 c.seq FROM c ORDER BY c.seq ASC`},
+ {"top_orderby_desc", `SELECT TOP 5 c.seq FROM c ORDER BY c.seq DESC`},
+ {"distinct_value_limit", `SELECT DISTINCT VALUE c.bucket FROM c OFFSET 0 LIMIT 3`},
+ {"distinct_offset_limit", `SELECT DISTINCT c.bucket FROM c OFFSET 2 LIMIT 3`},
+ {"distinct_top", `SELECT DISTINCT TOP 3 c.bucket FROM c`},
+ {"offset_limit", `SELECT c.seq FROM c OFFSET 2 LIMIT 3`},
+ {"agg_offset_limit", `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`},
+ {"min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 99999999`},
+ {"min_aliased_empty", `SELECT MIN(c.seq) AS m FROM c WHERE c.seq > 99999999`},
+ {"groupby_plain", `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket`},
+ } {
+ t.Run(tc.plan, func(t *testing.T) {
+ pipe, err := Default().CreateQueryPipeline(tc.query, recordedPlan(t, tc.plan), pkranges)
+ require.NoError(t, err)
+ pipe.Close()
+ })
+ }
+}
+
+// A GROUP BY that the user also ordered reaches the engine as a plan with no
+// order fields at all — the ordering survives only inside rewrittenQuery — so
+// the guard cannot see it. The group pipeline reads the direction off the
+// gateway's own per-partition streams instead.
+func TestCreateQueryPipelineAcceptsGroupByWithHiddenOrderBy(t *testing.T) {
+ raw := recordedPlan(t, "groupby_orderby")
+ plan, err := parsePlan([]byte(raw))
+ require.NoError(t, err)
+ assert.Empty(t, plan.QueryInfo.OrderBy)
+ assert.Empty(t, plan.QueryInfo.OrderByExpressions)
+ assert.Contains(t, plan.QueryInfo.RewrittenQuery, "ORDER BY")
+
+ pipe, err := Default().CreateQueryPipeline(
+ `SELECT c.bucket, COUNT(1) AS n FROM c GROUP BY c.bucket ORDER BY c.bucket`,
+ raw, `{"PartitionKeyRanges":[{"id":"0"}]}`)
+ require.NoError(t, err)
+ pipe.Close()
+}
+
+func TestRequireHandledWalksEveryPlanField(t *testing.T) {
+ n := 0
+ qi := planQueryInfo{RewrittenQuery: "SELECT 1", Top: &n}
+
+ require.NoError(t, requireHandled(qi, fieldRewrittenQuery, fieldTop))
+
+ err := requireHandled(qi, fieldRewrittenQuery)
+ require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+ assert.Contains(t, err.Error(), "queryInfo.Top", "TOP 0 is an instruction, not an absent field")
+}
+
+// A plan that spells a field out as empty asks for nothing, so it must not
+// make the dispatch reject a shape it can serve.
+func TestRequireHandledIgnoresFieldsSpeltOutEmpty(t *testing.T) {
+ p, err := parsePlan([]byte(`{"queryInfo":{"aggregates":[],"groupByAliases":[],` +
+ `"groupByAliasToAggregateType":{},"distinctType":"None","orderBy":[]}}`))
+ require.NoError(t, err)
+ require.NoError(t, requireHandled(p.QueryInfo))
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
index 42b070155e..e13f1ea13b 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
@@ -2,16 +2,16 @@ package gonative
import (
"encoding/json"
- "fmt"
+ "slices"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
"github.com/pkg/errors"
)
// groupByItem is one entry in the per-document groupByItems array. For a
-// single-key GROUP BY the array has one element; multi-key GROUP BY (not in
-// Stage 6's target inventory but straightforward to support since we hash the
-// whole array) would have more.
+// single-key GROUP BY the array has one element; multi-key GROUP BY would
+// have more, and needs no special handling here because the engine hashes and
+// orders the whole array.
type groupByItem struct {
Item json.RawMessage `json:"item"`
}
@@ -44,6 +44,9 @@ type groupState struct {
// report a given group populates these; subsequent partitions overwrite
// with the same value (Cosmos guarantees equality).
keyValues map[string]json.RawMessage
+ // items is the group's key as the gateway reported it, kept for ordering
+ // the output rows.
+ items []groupByItem
// aggregators are ordered per groupByAliases; entries whose alias is a
// group-key column are nil.
aggregators []aggregator
@@ -71,6 +74,20 @@ type groupPipeline struct {
// groups is the per-group state, keyed on the canonical JSON of
// groupByItems. Grows to the cardinality of distinct group keys.
groups map[string]*groupState
+ // order holds those keys in the order the groups were first seen, which
+ // is what the output falls back to when the keys cannot be ordered.
+ order []string
+
+ // descending says which way to order the output rows, and
+ // directionKnown whether the data has told us yet. A GROUP BY the user
+ // also ordered
+ // reaches the engine as a plan with no order fields at all — the ORDER BY
+ // survives only inside rewrittenQuery — so the direction can be read only
+ // off the responses: the gateway returns each partition's own groups in
+ // the requested order, and the first adjacent pair that differs says
+ // which way they run.
+ descending bool
+ directionKnown bool
// pending holds fully finalized rows awaiting the caller's next Run().
pending [][]byte
@@ -84,10 +101,9 @@ type groupPipeline struct {
// and groupByAliasToAggregateType fields; any alias missing from the map is
// treated as a group-key column.
//
-// Returns ErrUnsupportedPlanFeature when the plan composes GROUP BY with
-// operators this stage does not handle (ORDER BY, DISTINCT, TOP, OFFSET/LIMIT).
-// Those compositions are valid SQL but require wrapping groupPipeline in
-// additional operators — deferred.
+// Composing GROUP BY with TOP, OFFSET/LIMIT or DISTINCT would mean wrapping
+// this pipeline in another operator; CreateQueryPipeline rejects those plans
+// before they reach here.
func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error) {
qi := plan.QueryInfo
if len(qi.GroupByExpressions) == 0 {
@@ -96,13 +112,6 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error
if len(qi.GroupByAliases) == 0 {
return nil, errors.New("gonative: groupPipeline requires groupByAliases")
}
- // Stage 6 does not compose GROUP BY with other operators. Reject anything
- // that would require wrapping in ORDER BY / TOP / OFFSET-LIMIT / DISTINCT.
- if len(qi.OrderBy) > 0 || qi.Top != nil || qi.Offset != nil || qi.Limit != nil ||
- qi.DistinctType == "Unordered" || qi.DistinctType == "Ordered" {
- return nil, ErrUnsupportedPlanFeature
- }
-
aliases := append([]string(nil), qi.GroupByAliases...)
kinds := make([]string, len(aliases))
for i, alias := range aliases {
@@ -184,6 +193,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {
if err != nil {
return err
}
+ p.noteOrdering(resp.Documents)
for docIdx, doc := range resp.Documents {
if len(doc.GroupByItems) == 0 {
return errors.Errorf("gonative: group doc %d missing groupByItems", docIdx)
@@ -196,6 +206,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {
if !ok {
state = p.initGroupState(doc)
p.groups[key] = state
+ p.order = append(p.order, key)
}
// Merge this partition's aggregate contributions into the group's state.
for i, alias := range p.aliases {
@@ -229,6 +240,7 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {
func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {
s := &groupState{
keyValues: make(map[string]json.RawMessage, len(p.aliases)),
+ items: append([]groupByItem(nil), doc.GroupByItems...),
aggregators: make([]aggregator, len(p.aliases)),
}
for i, alias := range p.aliases {
@@ -244,13 +256,13 @@ func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {
return s
}
-// finalize walks the group map and produces one output row per group.
-// Emission order is unspecified (follows Go's map iteration randomization);
-// callers that need deterministic order should add an ORDER BY clause, which
-// a future composition stage will wrap around this pipeline.
+// finalize produces one output row per group, in group-key order — see
+// sortGroups. Walking p.groups instead would emit them in Go's randomized map
+// order, which made the same query answer in a different order every run.
func (p *groupPipeline) finalize() error {
- for _, state := range p.groups {
- row, err := p.buildRow(state)
+ p.sortGroups()
+ for _, key := range p.order {
+ row, err := p.buildRow(p.groups[key])
if err != nil {
return err
}
@@ -260,36 +272,106 @@ func (p *groupPipeline) finalize() error {
return nil
}
+// noteOrdering reads the output direction off one partition's response, which
+// is the only place a GROUP BY's ORDER BY reaches the engine. Gives up on a
+// key it cannot compare: sortGroups checks for those itself.
+func (p *groupPipeline) noteOrdering(docs []groupDoc) {
+ if p.directionKnown {
+ return
+ }
+ for i := 1; i < len(docs); i++ {
+ order, err := compareGroupKeys(docs[i-1].GroupByItems, docs[i].GroupByItems)
+ if err != nil {
+ return
+ }
+ if order != 0 {
+ p.descending = order > 0
+ p.directionKnown = true
+ return
+ }
+ }
+}
+
+// sortGroups puts p.order in group-key order, the direction taken from the
+// gateway's own per-partition streams. A GROUP BY without an ORDER BY has no
+// defined output order, so ordering those rows too costs nothing and is what
+// makes every run answer the same way.
+//
+// Leaves first-seen order in place when any group key is a shape Cosmos
+// cannot order (an array or an object), since the comparator would have to
+// invent an answer for those.
+func (p *groupPipeline) sortGroups() {
+ for _, key := range p.order {
+ for _, item := range p.groups[key].items {
+ if detectJSONType(item.Item) == jsonUnorderable {
+ return
+ }
+ }
+ }
+ slices.SortStableFunc(p.order, func(a, b string) int {
+ order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items)
+ if err != nil {
+ // Unreachable: every key was checked orderable just above.
+ return 0
+ }
+ if p.descending {
+ return -order
+ }
+ return order
+ })
+}
+
+// compareGroupKeys orders two group keys under Cosmos item ordering, column
+// by column.
+func compareGroupKeys(a, b []groupByItem) (int, error) {
+ for i := range min(len(a), len(b)) {
+ order, err := compareItems(a[i].Item, b[i].Item)
+ if err != nil {
+ return 0, err
+ }
+ if order != 0 {
+ return order, nil
+ }
+ }
+ return len(a) - len(b), nil
+}
+
// buildRow assembles the final JSON object for a single group, preserving
// the alias order from groupByAliases. JSON object key order in Go's encoder
// depends on sort for map marshaling, so we write the object manually.
+//
+// An alias whose aggregate has no value — MIN or MAX over a group where the
+// column is undefined in every row — is left out of the object, which is how
+// the gateway spells it.
func (p *groupPipeline) buildRow(s *groupState) ([]byte, error) {
var buf []byte
buf = append(buf, '{')
for i, alias := range p.aliases {
- if i > 0 {
- buf = append(buf, ',')
- }
- aliasJSON, err := json.Marshal(alias)
- if err != nil {
- return nil, err
- }
- buf = append(buf, aliasJSON...)
- buf = append(buf, ':')
+ value := json.RawMessage("null")
if p.kinds[i] == "" {
- v, ok := s.keyValues[alias]
- if !ok {
- buf = append(buf, []byte("null")...)
- continue
+ if v, ok := s.keyValues[alias]; ok {
+ value = v
}
- buf = append(buf, v...)
} else {
final, err := s.aggregators[i].finalize()
if err != nil {
return nil, err
}
- buf = append(buf, final...)
+ if final == nil {
+ continue
+ }
+ value = final
+ }
+ if len(buf) > 1 {
+ buf = append(buf, ',')
+ }
+ aliasJSON, err := json.Marshal(alias)
+ if err != nil {
+ return nil, err
}
+ buf = append(buf, aliasJSON...)
+ buf = append(buf, ':')
+ buf = append(buf, value...)
}
buf = append(buf, '}')
return buf, nil
@@ -323,26 +405,12 @@ func groupKey(items []groupByItem) (string, error) {
// "count":...}]}. For Count/Sum/Avg the aggregator consumes "item" directly.
// For Min/Max: prefer "item2" (the reliable cross-partition partial carrying
// count); fall back to synthesizing a single-count partial from "item" when
-// item2 is absent. Mirrors the feedAliasedPartial logic from Stage 1's
-// aliased-aggregate pipeline.
+// item2 is absent — see aggregatePartial, which both aggregate pipelines and
+// this one route through.
func extractAggregatePartial(kind string, raw json.RawMessage) (json.RawMessage, error) {
- var wrap aliasedItem
+ var wrap aggItem
if err := json.Unmarshal(raw, &wrap); err != nil {
return nil, errors.Wrap(err, "gonative: extract aggregate partial")
}
- switch kind {
- case "Min":
- if len(wrap.Item2) > 0 {
- return wrap.Item2, nil
- }
- return fmt.Appendf(nil, `{"min":%s,"count":1}`, wrap.Item), nil
- case "Max":
- if len(wrap.Item2) > 0 {
- return wrap.Item2, nil
- }
- return fmt.Appendf(nil, `{"max":%s,"count":1}`, wrap.Item), nil
- default:
- // Count, Sum, Avg — Item alone carries the partial.
- return wrap.Item, nil
- }
+ return aggregatePartial(kind, wrap), nil
}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
index e60dd2614c..bd4ffd12c9 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
@@ -103,28 +103,6 @@ func TestNewGroupPipeline_RequiresGroupByAliases(t *testing.T) {
assert.Contains(t, err.Error(), "groupByAliases")
}
-func TestNewGroupPipeline_RejectsCompositionWithOrderBy(t *testing.T) {
- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})
- plan.QueryInfo.OrderBy = []string{"Ascending"}
- _, err := newGroupPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
-}
-
-func TestNewGroupPipeline_RejectsCompositionWithTop(t *testing.T) {
- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})
- n := 5
- plan.QueryInfo.Top = &n
- _, err := newGroupPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
-}
-
-func TestNewGroupPipeline_RejectsCompositionWithDistinct(t *testing.T) {
- plan := newGroupPlan("c.a", []string{"a"}, map[string]string{"a": ""})
- plan.QueryInfo.DistinctType = "Unordered"
- _, err := newGroupPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
-}
-
// ---------------------------------------------------------- semantics
func TestGroupPipeline_SinglePartition_GroupKeyPassthrough(t *testing.T) {
@@ -352,3 +330,123 @@ func TestExtractAggregatePartial_MaxSynthesizesFromItemWhenItem2Absent(t *testin
require.NoError(t, err)
assert.JSONEq(t, `{"max":99,"count":1}`, string(got))
}
+
+// ---------------------------------------------------------- output order
+
+// groupKeysOf returns the group-key column of every emitted row, in the order
+// the pipeline emitted them.
+func groupKeysOf(t *testing.T, rows [][]byte, keyAlias string) []string {
+ t.Helper()
+ out := make([]string, 0, len(rows))
+ for _, r := range rows {
+ var obj map[string]any
+ require.NoError(t, json.Unmarshal(r, &obj))
+ v, ok := obj[keyAlias].(string)
+ require.True(t, ok, "row missing %s key: %s", keyAlias, r)
+ out = append(out, v)
+ }
+ return out
+}
+
+func countByCountryPlan() *planDoc {
+ return newGroupPlan("c.country",
+ []string{"country", "n"},
+ map[string]string{"country": "", "n": "Count"})
+}
+
+// A GROUP BY the user also ordered reaches the engine as a plan with no order
+// fields, so the direction comes off the data: the gateway returns each
+// partition's own groups in the requested order.
+func TestGroupPipeline_EmitsAscendingWhenThePartitionsRunAscending(t *testing.T) {
+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t,
+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},
+ ),
+ "1": mkGroupBody(t,
+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},
+ ),
+ })
+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))
+}
+
+func TestGroupPipeline_EmitsDescendingWhenThePartitionsRunDescending(t *testing.T) {
+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t,
+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},
+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
+ ),
+ "1": mkGroupBody(t,
+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},
+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
+ ),
+ })
+ assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))
+}
+
+// Without an ORDER BY the output order is unspecified, but it still has to be
+// the same on every run: before this it followed Go's map iteration and five
+// consecutive runs of one query gave five different orders.
+func TestGroupPipeline_OutputOrderIsStableAcrossRuns(t *testing.T) {
+ feed := map[string][]byte{
+ "0": mkGroupBody(t,
+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
+ groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},
+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
+ groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},
+ groupEntry{key: `"US"`, payload: `{"country":"US","n":{"item":9}}`},
+ ),
+ }
+ var first []string
+ for range 5 {
+ p, err := newGroupPipeline(countByCountryPlan(), []string{"0"})
+ require.NoError(t, err)
+ got := groupKeysOf(t, driveGroup(t, p, feed), "country")
+ if first == nil {
+ first = got
+ continue
+ }
+ assert.Equal(t, first, got)
+ }
+}
+
+// A group key Cosmos cannot order — an array or an object — leaves the output
+// in first-seen order rather than asking the comparator to invent one.
+func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {
+ plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})
+ p, err := newGroupPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t,
+ groupEntry{key: `["b"]`, payload: `{"tags":["b"],"n":{"item":1}}`},
+ groupEntry{key: `["a"]`, payload: `{"tags":["a"],"n":{"item":2}}`},
+ ),
+ })
+ require.Len(t, rows, 2)
+ assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[0]))
+ assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[1]))
+}
+
+// MIN over a group where the column is undefined in every row has no value,
+// and the gateway leaves that alias out of the row rather than writing null.
+func TestGroupPipeline_OmitsAnAliasWithNoValue(t *testing.T) {
+ plan := newGroupPlan("c.country", []string{"country", "lo"}, map[string]string{"country": "", "lo": "Min"})
+ p, err := newGroupPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t,
+ groupEntry{key: `"AE"`, payload: `{"country":"AE","lo":{"item2":{"count":0}}}`},
+ ),
+ })
+ require.Len(t, rows, 1)
+ assert.JSONEq(t, `{"country":"AE"}`, string(rows[0]))
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
index 1ac99cd4cf..5203d9af7e 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
@@ -19,8 +19,8 @@ import (
const orderByQueryFilterPlaceholder = "{documentdb-formattableorderbyquery-filter}"
// orderByItem is each element of the per-document orderByItems array. The
-// gateway emits one per ORDER BY expression; Stage 3 only supports single-key
-// ORDER BY so we read orderByItems[0].
+// gateway emits one per ORDER BY expression; the engine supports single-key
+// ORDER BY only, so it reads orderByItems[0].
type orderByItem struct {
Item json.RawMessage `json:"item"`
}
@@ -113,17 +113,22 @@ func (h *mergeHeap) Pop() any {
// ORDER BY expression. It emits rows in the requested order by k-way-merging
// each partition's gateway-sorted stream.
//
-// Stage 3 scope is single-key ORDER BY only. Multi-key ORDER BY requires a
-// Cosmos composite index on the container — if the container lacks one the
-// gateway rejects the query outright; if it has one the gateway emits a plan
-// with multiple entries in orderByExpressions, and the comparator would need
-// to walk all of them. Both cases are deferred.
+// Single-key ORDER BY only. Multi-key ORDER BY requires a Cosmos composite
+// index on the container — if the container lacks one the gateway rejects the
+// query outright; if it has one the gateway emits a plan with multiple entries
+// in orderByExpressions, and the comparator would need to walk all of them.
+// Both cases are deferred.
//
-// Stage 5 extends the pipeline with OFFSET/LIMIT awareness. Per Cosmos
-// semantics, OFFSET/LIMIT always composes with ORDER BY; the gateway rewrites
-// the per-partition query to `OFFSET 0 LIMIT (offset+limit)` so every
-// partition delivers enough rows to satisfy any interleaving, and this
-// pipeline handles the global skip + take after merging.
+// The operators ORDER BY composes with all apply to the merged stream, after
+// the k-way merge has put the rows in order:
+//
+// OFFSET/LIMIT — the gateway widens the per-partition query to
+// `OFFSET 0 LIMIT (offset+limit)` so every partition delivers enough rows
+// to cover any interleaving, and the global skip + take happens here.
+// TOP — the per-partition query carries `TOP n`, so each partition delivers
+// its own first n and the global first n is among them.
+// DISTINCT — the payload of each merged row is deduplicated against every
+// row already emitted, before OFFSET/LIMIT counts it.
//
// A range answers over as many pages as its share of the result needs. The
// merge therefore runs only while every range that still has pages has a row
@@ -139,22 +144,28 @@ type orderByPipeline struct {
closed bool
- // offset and limit implement plan.QueryInfo.Offset/Limit. A negative limit
- // (the sentinel returned by newOrderByPipeline when the plan has no limit)
- // means "unlimited". skipped counts rows dropped by offset; emitted counts
- // rows forwarded to the caller — both drive the skip+take logic in drain.
+ // offset and limit are the plan's global skip and take; a negative limit
+ // means unlimited. skipped counts rows dropped by offset; emitted counts
+ // rows forwarded to the caller — both drive the skip+take logic in merge.
offset int
limit int
skipped int
emitted int
+ // seen is the DISTINCT seen-set, nil unless the plan carries a DISTINCT.
+ // Keys are the canonical JSON of an emitted payload, as in
+ // distinctPipeline: a sorted stream makes duplicates adjacent, but the
+ // service reports distinctType "Unordered" for the object form even under
+ // ORDER BY, so the engine keeps the whole set rather than the last row.
+ seen map[string]struct{}
+
// pending emits produced by k-way merging when all partitions have data.
pending [][]byte
}
// newOrderByPipeline constructs a pipeline for a plan with exactly one ORDER BY
// expression. Returns ErrUnsupportedPlanFeature when the plan names multiple
-// ORDER BY keys — Stage 3 doesn't yet support that composition.
+// ORDER BY keys, which the comparator cannot walk.
func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, error) {
if len(plan.QueryInfo.OrderBy) != 1 {
return nil, ErrUnsupportedPlanFeature
@@ -167,21 +178,14 @@ func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, e
// the whole range and relies on the continuation token to resume it.
query := strings.ReplaceAll(plan.QueryInfo.RewrittenQuery, orderByQueryFilterPlaceholder, "true")
- // OFFSET/LIMIT always composes with ORDER BY per Cosmos syntax; defaults
- // mean "no skip, unlimited take".
- offset := 0
- if plan.QueryInfo.Offset != nil {
- if *plan.QueryInfo.Offset < 0 {
- return nil, errors.Errorf("gonative: negative OFFSET=%d", *plan.QueryInfo.Offset)
- }
- offset = *plan.QueryInfo.Offset
+ offset, limit, err := planSkipTake(plan.QueryInfo)
+ if err != nil {
+ return nil, err
}
- limit := -1
- if plan.QueryInfo.Limit != nil {
- if *plan.QueryInfo.Limit < 0 {
- return nil, errors.Errorf("gonative: negative LIMIT=%d", *plan.QueryInfo.Limit)
- }
- limit = *plan.QueryInfo.Limit
+
+ var seen map[string]struct{}
+ if plan.QueryInfo.DistinctType != "" {
+ seen = make(map[string]struct{})
}
partByID := make(map[string]int, len(pkRangeIDs))
@@ -198,6 +202,7 @@ func newOrderByPipeline(plan *planDoc, pkRangeIDs []string) (*orderByPipeline, e
partitions: partitions,
offset: offset,
limit: limit,
+ seen: seen,
}, nil
}
@@ -235,7 +240,9 @@ func (p *orderByPipeline) IsComplete() bool {
// 2. Otherwise ask for the next page of every range the merge is waiting on
// — the initial fan-out across all ranges on the first call.
func (p *orderByPipeline) Run() (*queryengine.PipelineResult, error) {
- p.merge()
+ if err := p.merge(); err != nil {
+ return nil, err
+ }
if len(p.pending) > 0 {
batch := p.pending
@@ -306,13 +313,14 @@ func (p *orderByPipeline) ProvideData(results []queryengine.QueryResult) error {
// still has pages ends the turn instead, because its next page may sort ahead
// of every other range's head.
//
-// OFFSET/LIMIT (Stage 5) apply here: the first `offset` rows are discarded,
-// and emission stops once `limit` rows have been collected (when limit >= 0).
-// Both are tracked on the pipeline so the next turn resumes where this one
-// stopped.
-func (p *orderByPipeline) merge() {
+// DISTINCT, OFFSET and LIMIT apply here, in that order: a row that repeats an
+// already-emitted payload is dropped before it counts against OFFSET, the
+// first `offset` surviving rows are discarded, and emission stops once `limit`
+// rows have been collected (when limit >= 0). All three are tracked on the
+// pipeline so the next turn resumes where this one stopped.
+func (p *orderByPipeline) merge() error {
if p.waitingForPage() {
- return
+ return nil
}
h := &mergeHeap{partitions: p.partitions, descending: p.descending}
for i, s := range p.partitions {
@@ -325,18 +333,25 @@ func (p *orderByPipeline) merge() {
for h.Len() > 0 {
if p.limit >= 0 && p.emitted >= p.limit {
// LIMIT reached — the caller has everything it asked for.
- return
+ return nil
}
top, ok := heap.Pop(h).(int)
if !ok {
// Unreachable: the heap is seeded with partition indexes only.
- return
+ return nil
}
s := p.partitions[top]
doc := s.pop()
- if p.skipped < p.offset {
+ fresh, err := p.firstSighting(doc.Payload)
+ if err != nil {
+ return err
+ }
+ switch {
+ case !fresh:
+ // DISTINCT already emitted this payload.
+ case p.skipped < p.offset:
p.skipped++
- } else {
+ default:
// Defensive copy — json.RawMessage aliases the underlying buffer.
p.pending = append(p.pending, append([]byte(nil), doc.Payload...))
p.emitted++
@@ -346,9 +361,28 @@ func (p *orderByPipeline) merge() {
continue
}
if !p.fanout.drained(s.id) {
- return
+ return nil
}
}
+ return nil
+}
+
+// firstSighting reports whether a DISTINCT lets this payload through, and
+// records it. Always true for a plan without a DISTINCT.
+func (p *orderByPipeline) firstSighting(payload json.RawMessage) (bool, error) {
+ if p.seen == nil {
+ return true, nil
+ }
+ canonical, err := canonicalJSON(payload)
+ if err != nil {
+ return false, err
+ }
+ key := string(canonical)
+ if _, ok := p.seen[key]; ok {
+ return false, nil
+ }
+ p.seen[key] = struct{}{}
+ return true, nil
}
// waitingForPage reports whether a range has nothing buffered and pages still
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
index 7a3d3cb69e..a886d62c12 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
@@ -501,3 +501,88 @@ func asStrings(rows [][]byte) []string {
}
return out
}
+
+// ------------------------------------------------- TOP and DISTINCT composed
+
+func newTopOrderByPipeline(t *testing.T, n int, pkRangeIDs ...string) *orderByPipeline {
+ t.Helper()
+ plan := &planDoc{}
+ plan.QueryInfo.OrderBy = []string{"Ascending"}
+ plan.QueryInfo.Top = intPtr(n)
+ plan.QueryInfo.RewrittenQuery = `SELECT TOP 5 c._rid, [{"item":c.x}] AS orderByItems, c AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x ASC`
+ p, err := newOrderByPipeline(plan, pkRangeIDs)
+ require.NoError(t, err)
+ return p
+}
+
+func newDistinctOrderByPipeline(t *testing.T, pkRangeIDs ...string) *orderByPipeline {
+ t.Helper()
+ plan := &planDoc{}
+ plan.QueryInfo.OrderBy = []string{"Ascending"}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.RewrittenQuery = `SELECT c._rid, [{"item":c.x}] AS orderByItems, {"x":c.x} AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x`
+ p, err := newOrderByPipeline(plan, pkRangeIDs)
+ require.NoError(t, err)
+ return p
+}
+
+// The per-partition query carries TOP n, so the merge sees up to n rows from
+// every range and the cap has to be applied again over the merged stream.
+func TestOrderByPipeline_TopCapsTheMergedStream(t *testing.T) {
+ p := newTopOrderByPipeline(t, 3, "0", "1")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `1`, payload: `{"x":1}`},
+ ordered{ob: `4`, payload: `{"x":4}`},
+ ordered{ob: `6`, payload: `{"x":6}`},
+ ),
+ "1": mkOrderByBody(t,
+ ordered{ob: `2`, payload: `{"x":2}`},
+ ordered{ob: `3`, payload: `{"x":3}`},
+ ordered{ob: `5`, payload: `{"x":5}`},
+ ),
+ })
+ assert.Equal(t, []string{`{"x":1}`, `{"x":2}`, `{"x":3}`}, rows)
+}
+
+// DISTINCT with an ORDER BY dedups the merged stream: the service reports
+// distinctType "Unordered" for the object form even under ORDER BY, and the
+// per-partition query carries no DISTINCT at all, so every duplicate arrives.
+func TestOrderByPipeline_DistinctDedupsTheMergedStream(t *testing.T) {
+ p := newDistinctOrderByPipeline(t, "0", "1")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"c"`, payload: `{"x":"c"}`},
+ ),
+ "1": mkOrderByBody(t,
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"b"`, payload: `{"x":"b"}`},
+ ordered{ob: `"c"`, payload: `{"x":"c"}`},
+ ),
+ })
+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`, `{"x":"c"}`}, rows)
+}
+
+// A duplicate must not spend a row of the LIMIT, or a paged DISTINCT comes
+// back short.
+func TestOrderByPipeline_DistinctDropsDuplicatesBeforeTheLimit(t *testing.T) {
+ plan := &planDoc{}
+ plan.QueryInfo.OrderBy = []string{"Ascending"}
+ plan.QueryInfo.DistinctType = "Unordered"
+ plan.QueryInfo.Limit = intPtr(2)
+ plan.QueryInfo.RewrittenQuery = `SELECT c._rid, [{"item":c.x}] AS orderByItems, {"x":c.x} AS payload FROM c WHERE ({documentdb-formattableorderbyquery-filter}) ORDER BY c.x OFFSET 0 LIMIT 2`
+ p, err := newOrderByPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `"b"`, payload: `{"x":"b"}`},
+ ),
+ })
+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`}, rows)
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go b/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go
index 6dee2a3904..70b5e23b1d 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/partition.go
@@ -13,7 +13,8 @@ import (
// VALUE form — plan.QueryInfo.Aggregates non-empty:
// {"Documents":[[{"item":18}]]}
// Each Documents[i] is an array of one item-object per aggregate in
-// plan.QueryInfo.Aggregates (positional match).
+// plan.QueryInfo.Aggregates (positional match). MIN/MAX carry item2 here
+// too: {"item":1,"item2":{"min":1,"count":5}}.
//
// Aliased form — plan.QueryInfo.GroupByAliasToAggregateType non-empty:
// {"Documents":[{"payload":{"alias":{"item":...,"item2":{...}}}}]}
@@ -21,15 +22,18 @@ import (
// keyed by alias name. Item2 is present for MIN/MAX (reliable partial
// {min|max, count}); for COUNT/SUM/AVG only Item is populated.
-// valueItem is each element inside a VALUE-form partition document.
-type valueItem struct {
- Item json.RawMessage `json:"item"`
+// aggItem is one aggregate's partial inside a partition document. Item is the
+// aggregate's own value; Item2 is the cross-partition partial the gateway adds
+// for MIN and MAX, which carries the count of rows the partition contributed.
+type aggItem struct {
+ Item json.RawMessage `json:"item"`
+ Item2 json.RawMessage `json:"item2,omitempty"`
}
// valuePartitionResponse decodes VALUE-form per-partition bodies.
// Documents[i] is an array whose length equals the number of aggregates.
type valuePartitionResponse struct {
- Documents [][]valueItem `json:"Documents"`
+ Documents [][]aggItem `json:"Documents"`
}
// parseValuePartitionResponse decodes a per-partition response body for a
@@ -42,17 +46,9 @@ func parseValuePartitionResponse(raw []byte) (*valuePartitionResponse, error) {
return &r, nil
}
-// aliasedItem is the per-alias partial inside an aliased-form partition
-// document's payload. Item is always present; Item2 is populated only for
-// MIN/MAX and carries the reliable cross-partition partial.
-type aliasedItem struct {
- Item json.RawMessage `json:"item"`
- Item2 json.RawMessage `json:"item2,omitempty"`
-}
-
// aliasedDoc is one entry of Documents for an aliased aggregate query.
type aliasedDoc struct {
- Payload map[string]aliasedItem `json:"payload"`
+ Payload map[string]aggItem `json:"payload"`
}
// aliasedPartitionResponse decodes aliased-form per-partition bodies.
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go
index 372cef18c5..2d0546546c 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline.go
@@ -2,7 +2,6 @@ package gonative
import (
"encoding/json"
- "fmt"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
"github.com/pkg/errors"
@@ -20,8 +19,16 @@ type aggPipelineBase struct {
rewrittenQuery string
fanout rangeFanout
aggregators []aggregator // len == number of aggregates to emit
+ kinds []string // positional: aggregate kind per aggregator
emitted bool
closed bool
+
+ // skip and take carry the plan's OFFSET and LIMIT. An aggregate query
+ // produces exactly one row, so they can only keep it or drop it; the
+ // gateway strips both from the rewritten query and leaves them here.
+ // take is negative when the plan sets no LIMIT.
+ skip int
+ take int
}
// requests returns the work to issue next: the initial fan-out across every
@@ -30,6 +37,39 @@ func (b *aggPipelineBase) requests() []queryengine.QueryRequest {
return b.fanout.requests(b.rewrittenQuery)
}
+// keepsRow reports whether the plan's OFFSET/LIMIT leaves the single
+// aggregate row in the result: OFFSET 1 skips past it and LIMIT 0 asks for
+// nothing, and the gateway answers both with no rows at all.
+func (b *aggPipelineBase) keepsRow() bool {
+ return b.skip == 0 && b.take != 0
+}
+
+// newAggPipelineBase builds the state both aggregate pipelines share from the
+// plan: the per-partition query, one aggregator per kind in positional order,
+// and the plan's OFFSET/LIMIT.
+func newAggPipelineBase(plan *planDoc, pkRangeIDs []string, kinds []string) (aggPipelineBase, error) {
+ aggs := make([]aggregator, len(kinds))
+ for i, kind := range kinds {
+ a := newAggregator(kind)
+ if a == nil {
+ return aggPipelineBase{}, errors.Errorf("gonative: unsupported aggregate %q", kind)
+ }
+ aggs[i] = a
+ }
+ skip, take, err := planSkipTake(plan.QueryInfo)
+ if err != nil {
+ return aggPipelineBase{}, err
+ }
+ return aggPipelineBase{
+ rewrittenQuery: plan.QueryInfo.RewrittenQuery,
+ fanout: newRangeFanout(pkRangeIDs),
+ aggregators: aggs,
+ kinds: kinds,
+ skip: skip,
+ take: take,
+ }, nil
+}
+
// -----------------------------------------------------------------------------
// VALUE-form pipeline — plan.QueryInfo.Aggregates non-empty, hasSelectValue true.
// Emits one final scalar (single aggregate) or one JSON array (multi-aggregate).
@@ -42,21 +82,11 @@ type valuePipeline struct {
// Returns an error if the plan references an aggregate kind the engine cannot
// handle, so the caller can surface ErrUnsupportedPlanFeature.
func newValuePipeline(plan *planDoc, pkRangeIDs []string) (*valuePipeline, error) {
- aggs := make([]aggregator, len(plan.QueryInfo.Aggregates))
- for i, kind := range plan.QueryInfo.Aggregates {
- a := newAggregator(kind)
- if a == nil {
- return nil, errors.Errorf("gonative: unsupported aggregate %q", kind)
- }
- aggs[i] = a
+ base, err := newAggPipelineBase(plan, pkRangeIDs, plan.QueryInfo.Aggregates)
+ if err != nil {
+ return nil, err
}
- return &valuePipeline{
- aggPipelineBase: aggPipelineBase{
- rewrittenQuery: plan.QueryInfo.RewrittenQuery,
- fanout: newRangeFanout(pkRangeIDs),
- aggregators: aggs,
- },
- }, nil
+ return &valuePipeline{aggPipelineBase: base}, nil
}
// Query returns the gateway-rewritten per-partition query.
@@ -90,6 +120,9 @@ func (p *valuePipeline) Run() (*queryengine.PipelineResult, error) {
return nil, err
}
p.emitted = true
+ if item == nil || !p.keepsRow() {
+ return &queryengine.PipelineResult{IsCompleted: true}, nil
+ }
return &queryengine.PipelineResult{Items: [][]byte{item}, IsCompleted: true}, nil
}
@@ -105,7 +138,7 @@ func (p *valuePipeline) ProvideData(results []queryengine.QueryResult) error {
return errors.Errorf("gonative: value partition document has %d items, expected %d", len(doc), len(p.aggregators))
}
for i, item := range doc {
- if err := p.aggregators[i].feedPartial(item.Item); err != nil {
+ if err := feedAggregatePartial(p.aggregators[i], p.kinds[i], item); err != nil {
return err
}
}
@@ -121,6 +154,11 @@ func (p *valuePipeline) ProvideData(results []queryengine.QueryResult) error {
// For single-aggregate queries it is the finalized scalar itself; for
// multi-aggregate queries (plan.QueryInfo.Aggregates with len > 1) it is a
// JSON array of finalized scalars in positional order.
+//
+// Returns nil when a single aggregate finalized to undefined — MIN, MAX or AVG
+// over an empty set — because the gateway answers those with no row. An
+// undefined slot inside a multi-aggregate array has no such spelling and goes
+// out as null.
func (p *valuePipeline) finalRow() ([]byte, error) {
finals := make([]json.RawMessage, len(p.aggregators))
for i, a := range p.aggregators {
@@ -128,6 +166,12 @@ func (p *valuePipeline) finalRow() ([]byte, error) {
if err != nil {
return nil, err
}
+ if v == nil {
+ if len(p.aggregators) == 1 {
+ return nil, nil
+ }
+ v = json.RawMessage(`null`)
+ }
finals[i] = v
}
if len(finals) == 1 {
@@ -143,7 +187,6 @@ func (p *valuePipeline) finalRow() ([]byte, error) {
type aliasedPipeline struct {
aggPipelineBase
aliases []string // preserved order from plan.QueryInfo.GroupByAliases
- kinds []string // positional: aggregate kind per alias
}
func newAliasedPipeline(plan *planDoc, pkRangeIDs []string) (*aliasedPipeline, error) {
@@ -152,28 +195,18 @@ func newAliasedPipeline(plan *planDoc, pkRangeIDs []string) (*aliasedPipeline, e
}
aliases := append([]string(nil), plan.QueryInfo.GroupByAliases...)
kinds := make([]string, len(aliases))
- aggs := make([]aggregator, len(aliases))
for i, alias := range aliases {
kind, ok := plan.QueryInfo.GroupByAliasToAggregateType[alias]
if !ok {
return nil, errors.Errorf("gonative: alias %q missing from groupByAliasToAggregateType", alias)
}
- a := newAggregator(kind)
- if a == nil {
- return nil, errors.Errorf("gonative: unsupported aggregate %q for alias %q", kind, alias)
- }
kinds[i] = kind
- aggs[i] = a
}
- return &aliasedPipeline{
- aggPipelineBase: aggPipelineBase{
- rewrittenQuery: plan.QueryInfo.RewrittenQuery,
- fanout: newRangeFanout(pkRangeIDs),
- aggregators: aggs,
- },
- aliases: aliases,
- kinds: kinds,
- }, nil
+ base, err := newAggPipelineBase(plan, pkRangeIDs, kinds)
+ if err != nil {
+ return nil, err
+ }
+ return &aliasedPipeline{aggPipelineBase: base, aliases: aliases}, nil
}
func (p *aliasedPipeline) Query() string { return p.rewrittenQuery }
@@ -195,6 +228,9 @@ func (p *aliasedPipeline) Run() (*queryengine.PipelineResult, error) {
return nil, err
}
p.emitted = true
+ if !p.keepsRow() {
+ return &queryengine.PipelineResult{IsCompleted: true}, nil
+ }
return &queryengine.PipelineResult{Items: [][]byte{item}, IsCompleted: true}, nil
}
@@ -210,7 +246,7 @@ func (p *aliasedPipeline) ProvideData(results []queryengine.QueryResult) error {
if !ok {
return errors.Errorf("gonative: aliased partition missing alias %q in payload", alias)
}
- if err := p.feedAliasedPartial(slot, p.kinds[slot], partial); err != nil {
+ if err := feedAggregatePartial(p.aggregators[slot], p.kinds[slot], partial); err != nil {
return err
}
}
@@ -222,32 +258,11 @@ func (p *aliasedPipeline) ProvideData(results []queryengine.QueryResult) error {
return nil
}
-// feedAliasedPartial routes an aliasedItem to the right aggregator. For
-// Min/Max it prefers Item2 (the reliable `{min|max, count}` partial) when
-// present, falling back to wrapping Item when the gateway short-circuited to
-// a single-partition shape (Item2 absent).
-func (p *aliasedPipeline) feedAliasedPartial(slot int, kind string, partial aliasedItem) error {
- switch kind {
- case "Min":
- if len(partial.Item2) > 0 {
- return p.aggregators[slot].feedPartial(partial.Item2)
- }
- synth := fmt.Appendf(nil, `{"min":%s,"count":1}`, partial.Item)
- return p.aggregators[slot].feedPartial(synth)
- case "Max":
- if len(partial.Item2) > 0 {
- return p.aggregators[slot].feedPartial(partial.Item2)
- }
- synth := fmt.Appendf(nil, `{"max":%s,"count":1}`, partial.Item)
- return p.aggregators[slot].feedPartial(synth)
- default:
- // Count, Sum, Avg — the Item itself carries the partial in the right shape.
- return p.aggregators[slot].feedPartial(partial.Item)
- }
-}
-
// finalRow builds `{"<alias1>": <v1>, ...}` preserving the original alias order
// from the plan (matters for deterministic output — map iteration is random).
+// An alias whose aggregate finalized to undefined is left out of the object,
+// which is how the gateway spells MIN, MAX or AVG over an empty set: the whole
+// row comes back as `{}`.
func (p *aliasedPipeline) finalRow() ([]byte, error) {
finals := make([]json.RawMessage, len(p.aggregators))
for i, a := range p.aggregators {
@@ -262,7 +277,10 @@ func (p *aliasedPipeline) finalRow() ([]byte, error) {
var buf []byte
buf = append(buf, '{')
for i, alias := range p.aliases {
- if i > 0 {
+ if finals[i] == nil {
+ continue
+ }
+ if len(buf) > 1 {
buf = append(buf, ',')
}
keyRaw, err := json.Marshal(alias)
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go
index 203fc34984..c994f640ec 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/pipeline_test.go
@@ -5,6 +5,8 @@ import (
"fmt"
"os"
"path/filepath"
+ "slices"
+ "strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
@@ -219,7 +221,7 @@ func TestAggPipeline_AliasedCountFoldsPartialsAcrossPages(t *testing.T) {
// drivePipelineAll runs a pipeline end-to-end against captured fixtures and
// returns every emitted row. Used for operators that emit more than one row
-// (DISTINCT, later Stage 3+ ORDER BY/TOP/LIMIT/GROUP BY).
+// (DISTINCT, ORDER BY, TOP, OFFSET/LIMIT, GROUP BY).
func drivePipelineAll(t *testing.T, queryID, originalQuery string) []json.RawMessage {
t.Helper()
base := filepath.Join("testdata", "query_"+queryID)
@@ -416,3 +418,109 @@ func indexOf(haystack, needle []byte) int {
}
return -1
}
+
+// drivePlan runs a recorded plan against the given per-range response bodies
+// and returns every emitted row. Unlike drivePipelineAll the bodies are given
+// inline, which is how the cases below pin an answer against a partition that
+// matched no row.
+func drivePlan(t *testing.T, planFile, query string, bodies map[string][]byte) []json.RawMessage {
+ t.Helper()
+ plan := mustReadFile(t, filepath.Join("testdata", "query_"+planFile+"_plan.json"))
+ ids := make([]string, 0, len(bodies))
+ for id := range bodies {
+ ids = append(ids, fmt.Sprintf("{\"id\":%q}", id))
+ }
+ // Range order decides which rows a window lands on, so it must not come
+ // from map iteration.
+ slices.Sort(ids)
+ pkranges := fmt.Sprintf(`{"PartitionKeyRanges":[%s]}`, strings.Join(ids, ","))
+
+ pipe, err := gonative.Default().CreateQueryPipeline(query, string(plan), pkranges)
+ require.NoError(t, err, "CreateQueryPipeline")
+ defer pipe.Close()
+
+ var rows []json.RawMessage
+ for i := 0; i < 50 && !pipe.IsComplete(); i++ {
+ res, err := pipe.Run()
+ require.NoError(t, err, "Run turn %d", i)
+ if len(res.Requests) > 0 {
+ results := make([]queryengine.QueryResult, 0, len(res.Requests))
+ for _, req := range res.Requests {
+ body, ok := bodies[req.PartitionKeyRangeID]
+ require.True(t, ok, "missing body for pk-range %s", req.PartitionKeyRangeID)
+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, body, ""))
+ }
+ require.NoError(t, pipe.ProvideData(results))
+ }
+ for _, it := range res.Items {
+ rows = append(rows, append(json.RawMessage(nil), it...))
+ }
+ }
+ require.True(t, pipe.IsComplete(), "pipeline did not converge")
+ return rows
+}
+
+// A VALUE-form MIN reads its partial from item2, the only one that reports how
+// many rows the partition contributed. Reading item instead fed a bare scalar
+// to an aggregator expecting `{min, count}`.
+func TestValuePipeline_MinReadsTheItem2Partial(t *testing.T) {
+ rows := drivePlan(t, "min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 0`,
+ map[string][]byte{
+ "0": []byte(`{"Documents":[[{"item":5,"item2":{"min":5,"count":3}}]]}`),
+ "1": []byte(`{"Documents":[[{"item":2,"item2":{"min":2,"count":7}}]]}`),
+ })
+ require.Len(t, rows, 1)
+ assert.JSONEq(t, `2`, string(rows[0]))
+}
+
+// MIN over a result set no row matched is undefined, and the gateway answers
+// it with no row at all. Before this the absent partial failed the query with
+// "gonative/agg.min: unexpected end of JSON input".
+func TestValuePipeline_MinOverAnEmptySetEmitsNoRow(t *testing.T) {
+ rows := drivePlan(t, "min_value_empty", `SELECT VALUE MIN(c.seq) FROM c WHERE c.seq > 99999999`,
+ map[string][]byte{
+ "0": []byte(`{"Documents":[[{"item2":{"count":0}}]]}`),
+ "1": []byte(`{"Documents":[[{"item2":{"count":0}}]]}`),
+ })
+ assert.Empty(t, rows)
+}
+
+// The aliased form of the same query answers with one row, the alias left out.
+func TestAliasedPipeline_MinOverAnEmptySetOmitsTheAlias(t *testing.T) {
+ rows := drivePlan(t, "min_aliased_empty", `SELECT MIN(c.seq) AS m FROM c WHERE c.seq > 99999999`,
+ map[string][]byte{
+ "0": []byte(`{"Documents":[{"payload":{"m":{"item2":{"count":0}}}}]}`),
+ "1": []byte(`{"Documents":[{"payload":{"m":{"item2":{"count":0}}}}]}`),
+ })
+ require.Len(t, rows, 1)
+ assert.JSONEq(t, `{}`, string(rows[0]))
+}
+
+// An aggregate produces exactly one row, so OFFSET 1 skips past it and the
+// gateway answers with nothing.
+func TestValuePipeline_OffsetPastTheOnlyRowEmitsNothing(t *testing.T) {
+ rows := drivePlan(t, "agg_offset_limit", `SELECT VALUE COUNT(1) FROM c OFFSET 1 LIMIT 1`,
+ map[string][]byte{
+ "0": []byte(`{"Documents":[[{"item":4}]]}`),
+ "1": []byte(`{"Documents":[[{"item":6}]]}`),
+ })
+ assert.Empty(t, rows)
+}
+
+// OFFSET/LIMIT with no ORDER BY cuts one window out of the concatenated
+// ranges. The engine used to reject the plan outright.
+func TestSkipTakePipeline_OffsetLimitWithoutOrderBy(t *testing.T) {
+ rows := drivePlan(t, "offset_limit", `SELECT c.seq FROM c OFFSET 2 LIMIT 3`,
+ map[string][]byte{
+ "0": []byte(`{"Documents":[{"seq":1},{"seq":2},{"seq":3}]}`),
+ })
+ assert.Equal(t, []string{`{"seq":3}`}, jsonStrings(rows))
+}
+
+func jsonStrings(rows []json.RawMessage) []string {
+ out := make([]string, 0, len(rows))
+ for _, r := range rows {
+ out = append(out, string(r))
+ }
+ return out
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go b/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go
index 9f544312e3..8d6fc7ae1e 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/plan.go
@@ -6,10 +6,7 @@ import (
"github.com/pkg/errors"
)
-// planDoc is the subset of the Cosmos query-plan document that Stage 1 uses.
-// Later stages (DISTINCT, ORDER BY, TOP, OFFSET/LIMIT, GROUP BY) will read
-// additional fields; declaring them defensively here keeps Stage 1 parsing
-// tolerant of plans that carry extra information.
+// planDoc is the subset of the Cosmos query-plan document the engine reads.
//
// Observed shapes (see queryengine/gonative/testdata):
//
@@ -30,22 +27,25 @@ type planDoc struct {
}
// planQueryInfo holds the instruction set the engine uses to build a pipeline.
+//
+// Every field here is an instruction, so the pipeline the dispatch picks must
+// claim each one the plan sets — see requireHandled. A field added to this
+// struct is therefore rejected until a pipeline claims it, which is what keeps
+// a new plan shape from losing an operator silently.
type planQueryInfo struct {
RewrittenQuery string `json:"rewrittenQuery"`
HasSelectValue bool `json:"hasSelectValue"`
Aggregates []string `json:"aggregates"`
GroupByAliases []string `json:"groupByAliases"`
GroupByAliasToAggregateType map[string]string `json:"groupByAliasToAggregateType"`
- // Fields that Stage 1 does not act on but decodes defensively so the
- // dispatch logic can reject unsupported plans cleanly.
- DistinctType string `json:"distinctType,omitempty"`
- Top *int `json:"top,omitempty"`
- Offset *int `json:"offset,omitempty"`
- Limit *int `json:"limit,omitempty"`
- OrderBy []string `json:"orderBy,omitempty"`
- OrderByExpressions []string `json:"orderByExpressions,omitempty"`
- GroupByExpressions []string `json:"groupByExpressions,omitempty"`
- HasNonStreamingOrderBy bool `json:"hasNonStreamingOrderBy,omitempty"`
+ DistinctType string `json:"distinctType,omitempty"`
+ Top *int `json:"top,omitempty"`
+ Offset *int `json:"offset,omitempty"`
+ Limit *int `json:"limit,omitempty"`
+ OrderBy []string `json:"orderBy,omitempty"`
+ OrderByExpressions []string `json:"orderByExpressions,omitempty"`
+ GroupByExpressions []string `json:"groupByExpressions,omitempty"`
+ HasNonStreamingOrderBy bool `json:"hasNonStreamingOrderBy,omitempty"`
}
// queryRange mirrors the gateway's pk-range description inside the plan.
@@ -56,15 +56,47 @@ type queryRange struct {
IsMaxInclusive bool `json:"isMaxInclusive"`
}
-// parsePlan decodes the gateway's query-plan JSON payload.
+// parsePlan decodes the gateway's query-plan JSON payload. It normalizes the
+// gateway's "None" distinctType to the empty string, so every caller can read
+// a set DistinctType as "this query has a DISTINCT".
func parsePlan(raw []byte) (*planDoc, error) {
var p planDoc
if err := json.Unmarshal(raw, &p); err != nil {
return nil, errors.Wrap(err, "gonative: parse query plan")
}
+ if p.QueryInfo.DistinctType == "None" {
+ p.QueryInfo.DistinctType = ""
+ }
return &p, nil
}
+// planSkipTake reads the plan's OFFSET, LIMIT and TOP as one global skip and
+// take, which is what every pipeline applies them as: the gateway strips them
+// from the rewritten query (or widens LIMIT to OFFSET+LIMIT) precisely so the
+// client applies them once, over the merged result.
+//
+// take is negative when the query names no upper bound. No Cosmos syntax
+// combines TOP with OFFSET/LIMIT, but a plan carrying both can only mean the
+// tighter of the two.
+func planSkipTake(qi planQueryInfo) (skip int, take int, err error) {
+ take = -1
+ for name, v := range map[string]*int{"OFFSET": qi.Offset, "LIMIT": qi.Limit, "TOP": qi.Top} {
+ if v != nil && *v < 0 {
+ return 0, 0, errors.Errorf("gonative: negative %s=%d", name, *v)
+ }
+ }
+ if qi.Offset != nil {
+ skip = *qi.Offset
+ }
+ if qi.Limit != nil {
+ take = *qi.Limit
+ }
+ if qi.Top != nil && (take < 0 || *qi.Top < take) {
+ take = *qi.Top
+ }
+ return skip, take, nil
+}
+
// pkRangesResponse decodes the gateway's pk-ranges response just enough to
// extract the ordered partition-key-range IDs the pipeline needs to fan out.
type pkRangesResponse struct {
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.json
new file mode 100644
index 0000000000..3ac05c3402
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_agg_offset_limit_plan.json
@@ -0,0 +1,30 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": 1,
+ "limit": 1,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [
+ "Count"
+ ],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT VALUE [{\"item\": COUNT(1)}]\nFROM c",
+ "hasSelectValue": true,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.json
new file mode 100644
index 0000000000..435c72f251
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_offset_limit_plan.json
@@ -0,0 +1,28 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "Unordered",
+ "top": null,
+ "offset": 2,
+ "limit": 3,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT DISTINCT c.bucket\nFROM c",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.json
new file mode 100644
index 0000000000..1597ef20a9
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_plan.json
@@ -0,0 +1,32 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "Unordered",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [
+ "Ascending"
+ ],
+ "orderByExpressions": [
+ "c.bucket"
+ ],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT c._rid, [{\"item\": c.bucket}] AS orderByItems, {\"bucket\": c.bucket} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.bucket",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.json
new file mode 100644
index 0000000000..420a174811
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_orderby_value_plan.json
@@ -0,0 +1,32 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "Ordered",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [
+ "Ascending"
+ ],
+ "orderByExpressions": [
+ "c.bucket"
+ ],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT c._rid, [{\"item\": c.bucket}] AS orderByItems, c.bucket AS payload\nFROM c\nWHERE (({documentdb-formattableorderbyquery-filter}) AND IS_DEFINED(c.bucket))\nORDER BY c.bucket",
+ "hasSelectValue": true,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.json
new file mode 100644
index 0000000000..da34c5a65e
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_top_plan.json
@@ -0,0 +1,28 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "Unordered",
+ "top": 3,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT DISTINCT c.bucket\nFROM c",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.json
new file mode 100644
index 0000000000..0095c198e5
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_distinct_value_limit_plan.json
@@ -0,0 +1,28 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "Unordered",
+ "top": null,
+ "offset": 0,
+ "limit": 3,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT DISTINCT VALUE c.bucket\nFROM c",
+ "hasSelectValue": true,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.json
new file mode 100644
index 0000000000..4c68e90482
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_offset_limit_plan.json
@@ -0,0 +1,36 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": 1,
+ "limit": 2,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [
+ "c.bucket"
+ ],
+ "groupByAliases": [
+ "bucket",
+ "n"
+ ],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {
+ "bucket": null,
+ "n": "Count"
+ },
+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.json
new file mode 100644
index 0000000000..260b3a4ea2
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_orderby_plan.json
@@ -0,0 +1,36 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [
+ "c.bucket"
+ ],
+ "groupByAliases": [
+ "bucket",
+ "n"
+ ],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {
+ "bucket": null,
+ "n": "Count"
+ },
+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket\nORDER BY c.bucket",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.json
new file mode 100644
index 0000000000..26af6f2a8a
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_plain_plan.json
@@ -0,0 +1,36 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [
+ "c.bucket"
+ ],
+ "groupByAliases": [
+ "bucket",
+ "n"
+ ],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {
+ "bucket": null,
+ "n": "Count"
+ },
+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.json
new file mode 100644
index 0000000000..18ab1b7cd8
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_groupby_top_plan.json
@@ -0,0 +1,36 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": 3,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [
+ "c.bucket"
+ ],
+ "groupByAliases": [
+ "bucket",
+ "n"
+ ],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {
+ "bucket": null,
+ "n": "Count"
+ },
+ "rewrittenQuery": "SELECT [{\"item\": c.bucket}] AS groupByItems, {\"bucket\": c.bucket, \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY c.bucket",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.json
new file mode 100644
index 0000000000..2bcbfdc8f0
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_aliased_empty_plan.json
@@ -0,0 +1,32 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [
+ "m"
+ ],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {
+ "m": "Min"
+ },
+ "rewrittenQuery": "SELECT {\"m\": {\"item\": MIN(c.seq), \"item2\": {\"min\": MIN(c.seq), \"count\": COUNT(c.seq)}}} AS payload\nFROM c\nWHERE (c.seq > 99999999)",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.json
new file mode 100644
index 0000000000..672b1f5f40
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_min_value_empty_plan.json
@@ -0,0 +1,30 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": null,
+ "limit": null,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [
+ "Min"
+ ],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT VALUE [{\"item\": MIN(c.seq), \"item2\": {\"min\": MIN(c.seq), \"count\": COUNT(c.seq)}}]\nFROM c\nWHERE (c.seq > 99999999)",
+ "hasSelectValue": true,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.json
new file mode 100644
index 0000000000..6be77dd13f
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_offset_limit_plan.json
@@ -0,0 +1,28 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": null,
+ "offset": 2,
+ "limit": 3,
+ "orderBy": [],
+ "orderByExpressions": [],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT c.seq\nFROM c\nOFFSET 0 LIMIT 5",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.json
new file mode 100644
index 0000000000..971941a48b
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_asc_plan.json
@@ -0,0 +1,32 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": 5,
+ "offset": null,
+ "limit": null,
+ "orderBy": [
+ "Ascending"
+ ],
+ "orderByExpressions": [
+ "c.seq"
+ ],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT TOP 5 c._rid, [{\"item\": c.seq}] AS orderByItems, {\"seq\": c.seq} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.seq ASC",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.json b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.json
new file mode 100644
index 0000000000..a04dc9d7f4
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/testdata/query_top_orderby_desc_plan.json
@@ -0,0 +1,32 @@
+{
+ "partitionedQueryExecutionInfoVersion": 2,
+ "queryInfo": {
+ "distinctType": "None",
+ "top": 5,
+ "offset": null,
+ "limit": null,
+ "orderBy": [
+ "Descending"
+ ],
+ "orderByExpressions": [
+ "c.seq"
+ ],
+ "groupByExpressions": [],
+ "groupByAliases": [],
+ "aggregates": [],
+ "groupByAliasToAggregateType": {},
+ "rewrittenQuery": "SELECT TOP 5 c._rid, [{\"item\": c.seq}] AS orderByItems, {\"seq\": c.seq} AS payload\nFROM c\nWHERE ({documentdb-formattableorderbyquery-filter})\nORDER BY c.seq DESC",
+ "hasSelectValue": false,
+ "dCountInfo": null,
+ "hasNonStreamingOrderBy": false
+ },
+ "queryRanges": [
+ {
+ "min": "",
+ "max": "FF",
+ "isMinInclusive": true,
+ "isMaxInclusive": false
+ }
+ ],
+ "hybridSearchQueryInfo": null
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go
index bf5fba17e1..160238df89 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake.go
@@ -7,10 +7,10 @@ import (
"github.com/pkg/errors"
)
-// topPartitionResponse decodes a "no rewrite" per-partition response — the
+// topPartitionResponse decodes an unwrapped per-partition response — the
// Documents array carries raw user-shaped rows directly (no orderByItems /
-// payload wrapping). Used for standalone TOP, where the gateway runs the
-// original query per partition and returns whatever matches.
+// payload wrapping). Used for TOP and for OFFSET/LIMIT without an ORDER BY,
+// where each partition returns whatever matches and the engine cuts the page.
type topPartitionResponse struct {
Documents []json.RawMessage `json:"Documents"`
}
@@ -23,68 +23,76 @@ func parseTopPartitionResponse(raw []byte) (*topPartitionResponse, error) {
return &r, nil
}
-// topPipeline implements queryengine.QueryPipeline for standalone TOP queries
-// — plans with `top: N` and none of ORDER BY, DISTINCT, aggregates, or GROUP BY.
-// Rows from each partition are concatenated and the first N are emitted.
+// skipTakePipeline implements queryengine.QueryPipeline for the plans whose
+// only operators cut a window out of an unordered result: `TOP n`, and
+// `OFFSET n LIMIT m` with no ORDER BY. Rows from each partition are
+// concatenated, the first `skip` are dropped and the next `take` emitted.
//
-// TOP composed with ORDER BY/DISTINCT/aggregates/GROUP BY is out of Stage 4's
-// scope; the dispatch rejects those plans with ErrUnsupportedPlanFeature.
+// Which rows those are is unspecified without an ORDER BY, which is what
+// makes the window safe to cut here: the gateway widens the per-partition
+// query to `OFFSET 0 LIMIT (offset+limit)` (or leaves TOP n on it), so every
+// range offers at least as many rows as the window needs and no row the
+// window should hold is missing from the union.
//
-// A TOP larger than a response page needs several pages from a range. The
-// pipeline keeps asking for them until it holds N rows, at which point it
+// A window larger than a response page needs several pages from a range. The
+// pipeline keeps asking for them until it holds take rows, at which point it
// stops: IsComplete trips on the emitted count and no further page is
// requested, however many rows the ranges still hold.
-type topPipeline struct {
+type skipTakePipeline struct {
query string
- n int
+ skip int
+ take int // negative means unlimited
fanout rangeFanout
closed bool
- // pending items ready to flush to the caller, already capped at N total.
+ // pending items ready to flush to the caller, already capped at take.
pending [][]byte
+ // skipped counts the rows dropped by skip so far.
+ skipped int
// emitted tracks how many rows have been drained via Run(); we stop
- // accepting/emitting once we hit N.
+ // accepting/emitting once we hit take.
emitted int
// accumulated counts pending + emitted. Keeps ProvideData from growing
- // `pending` past N so downstream flushes never over-emit.
+ // `pending` past take so downstream flushes never over-emit.
accumulated int
}
-// newTopPipeline builds a pipeline for a plan whose only operator is TOP.
-// Returns ErrUnsupportedPlanFeature when the plan carries any other operator
-// that would require composition (ORDER BY, DISTINCT, aggregate, GROUP BY).
-func newTopPipeline(plan *planDoc, pkRangeIDs []string) (*topPipeline, error) {
- if plan.QueryInfo.Top == nil {
- return nil, errors.New("gonative: topPipeline requires a non-nil Top")
+// newSkipTakePipeline builds a pipeline for a plan whose only operators are
+// TOP and/or OFFSET/LIMIT. Composition with another operator is the
+// dispatch's call, not this constructor's: CreateQueryPipeline rejects a plan
+// carrying a field this pipeline cannot honour before it gets here.
+func newSkipTakePipeline(plan *planDoc, pkRangeIDs []string, query string) (*skipTakePipeline, error) {
+ if plan.QueryInfo.Top == nil && plan.QueryInfo.Offset == nil && plan.QueryInfo.Limit == nil {
+ return nil, errors.New("gonative: skipTakePipeline requires TOP, OFFSET or LIMIT")
}
- // Composition with other operators is a follow-up. Stage 4 only handles
- // standalone TOP — the dispatcher routes composed plans elsewhere or
- // rejects them.
- if len(plan.QueryInfo.OrderBy) > 0 ||
- plan.QueryInfo.DistinctType == "Unordered" || plan.QueryInfo.DistinctType == "Ordered" ||
- len(plan.QueryInfo.Aggregates) > 0 ||
- len(plan.QueryInfo.GroupByAliasToAggregateType) > 0 ||
- len(plan.QueryInfo.GroupByExpressions) > 0 {
- return nil, ErrUnsupportedPlanFeature
+ skip, take, err := planSkipTake(plan.QueryInfo)
+ if err != nil {
+ return nil, err
}
- n := *plan.QueryInfo.Top
- if n < 0 {
- return nil, errors.Errorf("gonative: negative TOP N=%d", n)
+ // A bare TOP plan carries no rewrittenQuery, so each range answers the
+ // user's own query, TOP and all. An OFFSET/LIMIT plan carries one widened
+ // to `OFFSET 0 LIMIT (offset+limit)`, and running that is what lets this
+ // pipeline cut the window globally rather than per range.
+ if plan.QueryInfo.RewrittenQuery != "" {
+ query = plan.QueryInfo.RewrittenQuery
}
- return &topPipeline{
- n: n,
+ return &skipTakePipeline{
+ query: query,
+ skip: skip,
+ take: take,
fanout: newRangeFanout(pkRangeIDs),
}, nil
}
// Query implements queryengine.QueryPipeline.
-func (p *topPipeline) Query() string { return p.query }
+func (p *skipTakePipeline) Query() string { return p.query }
-// IsComplete returns true once either (a) we've emitted N rows — which covers
-// TOP 0, so the pipeline never fires a request for it — or (b) every range has
-// delivered its last page and all pending rows have been flushed.
-func (p *topPipeline) IsComplete() bool {
- if p.emitted >= p.n {
+// IsComplete returns true once either (a) we've emitted take rows — which
+// covers TOP 0 and LIMIT 0, so the pipeline never fires a request for them —
+// or (b) every range has delivered its last page and all pending rows have
+// been flushed.
+func (p *skipTakePipeline) IsComplete() bool {
+ if p.take >= 0 && p.emitted >= p.take {
return true
}
if len(p.pending) > 0 {
@@ -94,12 +102,12 @@ func (p *topPipeline) IsComplete() bool {
}
// Close implements queryengine.QueryPipeline.
-func (p *topPipeline) Close() { p.closed = true }
+func (p *skipTakePipeline) Close() { p.closed = true }
// Run drives one pipeline turn: flush the rows already in hand, then ask for
// the next page of every range that has one — the initial fan-out on the first
// call.
-func (p *topPipeline) Run() (*queryengine.PipelineResult, error) {
+func (p *skipTakePipeline) Run() (*queryengine.PipelineResult, error) {
if len(p.pending) > 0 {
batch := p.pending
p.pending = nil
@@ -112,14 +120,14 @@ func (p *topPipeline) Run() (*queryengine.PipelineResult, error) {
return &queryengine.PipelineResult{Requests: p.fanout.requests(p.query)}, nil
}
-// ProvideData consumes each partition's response and appends rows to `pending`
-// up to the overall TOP N limit. Anything beyond N is dropped.
-func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {
+// ProvideData consumes each partition's response, drops the first `skip` rows
+// and appends the rest to `pending` up to `take` rows in all.
+func (p *skipTakePipeline) ProvideData(results []queryengine.QueryResult) error {
for _, r := range results {
if err := p.fanout.note(r); err != nil {
return err
}
- if p.accumulated >= p.n {
+ if p.full() {
// Already have enough rows — leave this response unread.
continue
}
@@ -128,9 +136,13 @@ func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {
return err
}
for _, doc := range resp.Documents {
- if p.accumulated >= p.n {
+ if p.full() {
break
}
+ if p.skipped < p.skip {
+ p.skipped++
+ continue
+ }
// Defensive copy — json.RawMessage aliases the input buffer.
p.pending = append(p.pending, append([]byte(nil), doc...))
p.accumulated++
@@ -138,3 +150,8 @@ func (p *topPipeline) ProvideData(results []queryengine.QueryResult) error {
}
return nil
}
+
+// full reports whether the window already holds every row it can take.
+func (p *skipTakePipeline) full() bool {
+ return p.take >= 0 && p.accumulated >= p.take
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go
index 55bce60cb0..d0c5ae6ecd 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/topskiptake_test.go
@@ -30,13 +30,28 @@ func newTopPlan(n int) *planDoc {
return p
}
-// driveTop feeds the given per-partition bodies through a TOP pipeline and
-// returns the emitted rows.
-func driveTop(t *testing.T, p *topPipeline, feed map[string][]byte) [][]byte {
+func newOffsetLimitPlan(offset, limit int) *planDoc {
+ p := &planDoc{}
+ p.QueryInfo.Offset = intPtr(offset)
+ p.QueryInfo.Limit = intPtr(limit)
+ p.QueryInfo.RewrittenQuery = `SELECT c.name FROM c OFFSET 0 LIMIT 5`
+ return p
+}
+
+func newSkipTake(t *testing.T, plan *planDoc, pkRangeIDs ...string) *skipTakePipeline {
+ t.Helper()
+ p, err := newSkipTakePipeline(plan, pkRangeIDs, `SELECT * FROM c`)
+ require.NoError(t, err)
+ return p
+}
+
+// driveTop feeds the given per-partition bodies through a skip/take pipeline
+// and returns the emitted rows.
+func driveTop(t *testing.T, p *skipTakePipeline, feed map[string][]byte) [][]byte {
t.Helper()
res, err := p.Run()
require.NoError(t, err)
- if p.n == 0 {
+ if p.take == 0 {
// Edge case: TOP 0 completes immediately without any partition work.
assert.Empty(t, res.Requests)
return nil
@@ -62,46 +77,31 @@ func driveTop(t *testing.T, p *topPipeline, feed map[string][]byte) [][]byte {
// ---------------------------------------------------------------- validation
-func TestNewTopPipeline_RequiresNonNilTop(t *testing.T) {
- plan := &planDoc{}
- _, err := newTopPipeline(plan, []string{"0"})
+func TestNewSkipTakePipeline_RequiresAWindow(t *testing.T) {
+ _, err := newSkipTakePipeline(&planDoc{}, []string{"0"}, `SELECT * FROM c`)
require.Error(t, err)
- assert.Contains(t, err.Error(), "non-nil Top")
+ assert.Contains(t, err.Error(), "requires TOP, OFFSET or LIMIT")
}
-func TestNewTopPipeline_RejectsNegativeN(t *testing.T) {
- plan := newTopPlan(-1)
- _, err := newTopPipeline(plan, []string{"0"})
+func TestNewSkipTakePipeline_RejectsNegativeN(t *testing.T) {
+ _, err := newSkipTakePipeline(newTopPlan(-1), []string{"0"}, `SELECT * FROM c`)
require.Error(t, err)
assert.Contains(t, err.Error(), "negative TOP")
}
-func TestNewTopPipeline_RejectsCompositionWithOrderBy(t *testing.T) {
- plan := newTopPlan(5)
- plan.QueryInfo.OrderBy = []string{"Ascending"}
- _, err := newTopPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
-}
-
-func TestNewTopPipeline_RejectsCompositionWithDistinct(t *testing.T) {
- plan := newTopPlan(5)
- plan.QueryInfo.DistinctType = "Unordered"
- _, err := newTopPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
-}
-
-func TestNewTopPipeline_RejectsCompositionWithAggregate(t *testing.T) {
- plan := newTopPlan(5)
- plan.QueryInfo.Aggregates = []string{"Count"}
- _, err := newTopPipeline(plan, []string{"0"})
- require.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+// A bare TOP plan carries no rewrittenQuery, so the range answers the user's
+// own query; an OFFSET/LIMIT plan carries one widened to cover the window.
+func TestNewSkipTakePipeline_QueryComesFromThePlanWhenRewritten(t *testing.T) {
+ assert.Equal(t, `SELECT * FROM c`, newSkipTake(t, newTopPlan(3), "0").Query())
+ assert.Equal(t, `SELECT c.name FROM c OFFSET 0 LIMIT 5`,
+ newSkipTake(t, newOffsetLimitPlan(2, 3), "0").Query())
}
// ---------------------------------------------------------------- semantics
func TestTopPipeline_SinglePartition_EmitsFirstN(t *testing.T) {
plan := newTopPlan(3)
- p, err := newTopPipeline(plan, []string{"0"})
+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)
require.NoError(t, err)
rows := driveTop(t, p, map[string][]byte{
@@ -114,7 +114,7 @@ func TestTopPipeline_SinglePartition_EmitsFirstN(t *testing.T) {
func TestTopPipeline_FewerRowsThanN_EmitsAll(t *testing.T) {
plan := newTopPlan(10)
- p, err := newTopPipeline(plan, []string{"0"})
+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)
require.NoError(t, err)
rows := driveTop(t, p, map[string][]byte{
@@ -126,7 +126,7 @@ func TestTopPipeline_FewerRowsThanN_EmitsAll(t *testing.T) {
func TestTopPipeline_MultiPartition_CapsAtN(t *testing.T) {
plan := newTopPlan(4)
- p, err := newTopPipeline(plan, []string{"A", "B", "C"})
+ p, err := newSkipTakePipeline(plan, []string{"A", "B", "C"}, `SELECT * FROM c`)
require.NoError(t, err)
rows := driveTop(t, p, map[string][]byte{
@@ -139,7 +139,7 @@ func TestTopPipeline_MultiPartition_CapsAtN(t *testing.T) {
func TestTopPipeline_ZeroN_EmitsNothing(t *testing.T) {
plan := newTopPlan(0)
- p, err := newTopPipeline(plan, []string{"0"})
+ p, err := newSkipTakePipeline(plan, []string{"0"}, `SELECT * FROM c`)
require.NoError(t, err)
rows := driveTop(t, p, nil)
@@ -149,7 +149,7 @@ func TestTopPipeline_ZeroN_EmitsNothing(t *testing.T) {
func TestTopPipeline_EmptyPartitions(t *testing.T) {
plan := newTopPlan(5)
- p, err := newTopPipeline(plan, []string{"A", "B"})
+ p, err := newSkipTakePipeline(plan, []string{"A", "B"}, `SELECT * FROM c`)
require.NoError(t, err)
rows := driveTop(t, p, map[string][]byte{
@@ -162,8 +162,7 @@ func TestTopPipeline_EmptyPartitions(t *testing.T) {
// A TOP larger than a page reads as many pages as it takes to collect N rows.
func TestTopPipeline_ReadsPagesUntilNRowsAreInHand(t *testing.T) {
- p, err := newTopPipeline(newTopPlan(5), []string{"0"})
- require.NoError(t, err)
+ p := newSkipTake(t, newTopPlan(5), "0")
rows, read := drivePages(t, p, map[string][]page{
"0": {
@@ -179,8 +178,7 @@ func TestTopPipeline_ReadsPagesUntilNRowsAreInHand(t *testing.T) {
// Once N rows are in hand the pipeline stops, however many pages the ranges
// still hold.
func TestTopPipeline_StopsAtNWithoutReadingTheNextPage(t *testing.T) {
- p, err := newTopPipeline(newTopPlan(4), []string{"0"})
- require.NoError(t, err)
+ p := newSkipTake(t, newTopPlan(4), "0")
rows, read := drivePages(t, p, map[string][]page{
"0": {
@@ -192,3 +190,40 @@ func TestTopPipeline_StopsAtNWithoutReadingTheNextPage(t *testing.T) {
assert.Len(t, rows, 4)
assert.Equal(t, map[string]int{"0": 2}, read)
}
+
+// ------------------------------------------------- OFFSET/LIMIT, no ORDER BY
+
+// OFFSET/LIMIT without an ORDER BY is ordinary result paging. The gateway
+// widens the per-partition query to cover the window, so every range offers
+// at least as many rows as the window needs and the cut happens here.
+func TestSkipTakePipeline_OffsetLimitCutsOneWindowGlobally(t *testing.T) {
+ p := newSkipTake(t, newOffsetLimitPlan(2, 3), "0", "1")
+ rows := driveTop(t, p, map[string][]byte{
+ "0": mkTopBody(t, `{"x":1}`, `{"x":2}`, `{"x":3}`),
+ "1": mkTopBody(t, `{"x":4}`, `{"x":5}`, `{"x":6}`),
+ })
+ assert.Equal(t, []string{`{"x":3}`, `{"x":4}`, `{"x":5}`}, stringsOf(rows))
+}
+
+func TestSkipTakePipeline_OffsetPastEveryRowEmitsNothing(t *testing.T) {
+ p := newSkipTake(t, newOffsetLimitPlan(10, 3), "0")
+ rows := driveTop(t, p, map[string][]byte{
+ "0": mkTopBody(t, `{"x":1}`, `{"x":2}`),
+ })
+ assert.Empty(t, rows)
+ assert.True(t, p.IsComplete())
+}
+
+// A window that straddles a page boundary has to keep reading the range.
+func TestSkipTakePipeline_OffsetLimitReadsPagesUntilTheWindowIsFull(t *testing.T) {
+ p := newSkipTake(t, newOffsetLimitPlan(3, 2), "0")
+ rows, read := drivePages(t, p, map[string][]page{
+ "0": {
+ {body: mkTopBody(t, `{"x":1}`, `{"x":2}`), continuation: "0-page-2"},
+ {body: mkTopBody(t, `{"x":3}`, `{"x":4}`), continuation: "0-page-3"},
+ {body: mkTopBody(t, `{"x":5}`, `{"x":6}`)},
+ },
+ })
+ assert.Equal(t, []string{`{"x":4}`, `{"x":5}`}, stringsOf(rows))
+ assert.Equal(t, map[string]int{"0": 3}, read)
+}