Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(cosmosdb): group and order cross-partition queries the way the service does (#21897)

Danny Xu5f5257e2c8d1
diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.go
index 2a14ad5f3a..9f4f6edd24 100644
--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go
+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go
@@ -642,6 +642,43 @@ func pagedRows() int {
return len(pagedBigBuckets)*pagedBigRows + len(pagedSmallBuckets)*pagedSmallRows
}
+// pagedShapes gives eleven of the fixture's rows a `v`, one per shape a
+// document can hold, chosen so that the values span the container's ranges:
+// two rows in each big bucket and one in each small one. Every other row
+// leaves `v` undefined, so a query grouping or ordering on it meets both.
+//
+// Ordering them is what the service does by type — undefined < null < bool <
+// number < string < array < object — and the two seqs inside each of the array
+// and object classes are interchangeable, since the service orders those by
+// its index's own encoding.
+var pagedShapes = []struct {
+ seq int
+ value string
+}{
+ {seq: 0, value: `null`},
+ {seq: 1, value: `false`},
+ {seq: pagedBigRows, value: `true`},
+ {seq: pagedBigRows + 1, value: `1`},
+ {seq: 2 * pagedBigRows, value: `20`},
+ {seq: 2*pagedBigRows + 1, value: `"a"`},
+ {seq: 2*pagedBigRows + pagedSmallRows, value: `"b"`},
+ {seq: 2*pagedBigRows + 2*pagedSmallRows, value: `[1,2]`},
+ {seq: 2*pagedBigRows + 3*pagedSmallRows, value: `[2]`},
+ {seq: 2*pagedBigRows + 4*pagedSmallRows, value: `{"a":1}`},
+ {seq: 2*pagedBigRows + 5*pagedSmallRows, value: `{"b":1}`},
+}
+
+// pagedShapeOf returns the shape attached to a row, or "" when the row leaves
+// `v` undefined.
+func pagedShapeOf(seq int) string {
+ for _, shape := range pagedShapes {
+ if shape.seq == seq {
+ return shape.value
+ }
+ }
+ return ""
+}
+
var (
pagedOnce sync.Once
pagedContainerClient *azcosmos.ContainerClient
@@ -712,11 +749,15 @@ func seedPaged(driver *Driver) (*azcosmos.ContainerClient, int, error) {
for range 12 {
wg.Go(func() {
for r := range rows {
- item, err := json.Marshal(pagedRow{
+ row := pagedRow{
ID: fmt.Sprintf("row-%d", r.seq),
Bucket: r.bucket,
Seq: r.seq,
- })
+ }
+ if shape := pagedShapeOf(r.seq); shape != "" {
+ row.V = json.RawMessage(shape)
+ }
+ item, err := json.Marshal(row)
if err == nil {
err = seedItem(ctx, container, r.bucket, item)
}
@@ -748,6 +789,10 @@ type pagedRow struct {
ID string `json:"id"`
Bucket string `json:"bucket"`
Seq int `json:"seq"`
+ // V carries one of pagedShapes on the rows that have one, and is absent
+ // from every other document — which is what makes the fixture answer for
+ // a column a schemaless store defines on only some of its rows.
+ V json.RawMessage `json:"v,omitempty"`
}
// pagedSeqs runs a query against the multi-page fixture and returns the seq of
@@ -1275,3 +1320,56 @@ func TestCrossPartitionDistinctWithOrderByThroughTheDriver(t *testing.T) {
}
assert.Equal(t, slices.Sorted(slices.Values(crossPartitionCountries)), got)
}
+
+// TestCrossPartitionSparseGroupBy groups on a column the documents do not all
+// define, which is ordinary in a schemaless store. The service answers the
+// undefined rows as a group of their own, counted apart from an explicit null
+// and spelled with the alias left out of the row.
+func TestCrossPartitionSparseGroupBy(t *testing.T) {
+ container := pagedFixture(t)
+
+ want := []string{fmt.Sprintf(`{"n":%d}`, pagedRows()-len(pagedShapes))}
+ for _, shape := range pagedShapes {
+ want = append(want, fmt.Sprintf(`{"k":%s,"n":1}`, shape.value))
+ }
+ rows := pagedQuery(t, container, `SELECT c.v AS k, COUNT(1) AS n FROM c GROUP BY c.v`)
+ assert.ElementsMatch(t, want, rows)
+
+ // The groups also come back in key order, the undefined one first, which
+ // is what makes two runs of one query agree.
+ require.NotEmpty(t, rows)
+ assert.Equal(t, want[0], rows[0])
+ assert.ElementsMatch(t, []string{`{"k":{"a":1},"n":1}`, `{"k":{"b":1},"n":1}`}, rows[len(rows)-2:],
+ "objects sort last")
+}
+
+// TestCrossPartitionMixedTypeOrder checks ORDER BY against the order the
+// service itself returns: undefined < null < bool < number < string < array <
+// object.
+//
+// Two different arrays, or two different objects, it cannot order when they
+// reach the merge from different ranges, because the service orders those by
+// its index's own encoding — [1,2] < [2] < [1,3] on a real container. Which
+// range holds which value is the emulator's to decide, so the engine's refusal
+// is covered in merge_test.go, where the ranges are given.
+func TestCrossPartitionMixedTypeOrder(t *testing.T) {
+ container := pagedFixture(t)
+
+ scalars := make([]int, 0, len(pagedShapes))
+ for _, shape := range pagedShapes {
+ if shape.value[0] != '[' && shape.value[0] != '{' {
+ scalars = append(scalars, shape.seq)
+ }
+ }
+ got := pagedSeqs(t, container,
+ `SELECT c.seq FROM c WHERE IS_DEFINED(c.v) AND NOT IS_ARRAY(c.v) AND NOT IS_OBJECT(c.v) ORDER BY c.v`)
+ assert.Equal(t, scalars, got, "the scalar classes are ordered as the service orders them")
+
+ // One composite among them still orders, since the class ranking is the
+ // service's own: an array sorts after every string.
+ withOneArray := append(slices.Clone(scalars), pagedShapes[7].seq)
+ got = pagedSeqs(t, container, fmt.Sprintf(
+ `SELECT c.seq FROM c WHERE IS_DEFINED(c.v) AND NOT IS_OBJECT(c.v) AND (NOT IS_ARRAY(c.v) OR c.seq = %d) ORDER BY c.v`,
+ pagedShapes[7].seq))
+ assert.Equal(t, withOneArray, got, "an array sorts after the scalars")
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
index a08a7ddfdb..2f8ca69659 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go
@@ -202,3 +202,21 @@ func TestNumberRawMessage_IntegerFormatting(t *testing.T) {
assert.Equal(t, json.RawMessage(`0`), numberRawMessage(0))
assert.Equal(t, json.RawMessage(`1.5`), numberRawMessage(1.5))
}
+
+// MIN and MAX fold their partials through compareItems, so the composite
+// classes reach them too: two ranges reporting the same array agree on the
+// answer, while two reporting different arrays have none the values can give.
+func TestMinAgg_FoldsEqualComposites(t *testing.T) {
+ out := feedFinal(t, newAggregator("Min"),
+ []byte(`{"min":[1,2],"count":2}`),
+ []byte(`{"min":[1,2],"count":3}`),
+ )
+ assert.Equal(t, json.RawMessage(`[1,2]`), out)
+}
+
+func TestMinAgg_RefusesTwoDifferentComposites(t *testing.T) {
+ a := newAggregator("Min")
+ require.NoError(t, a.feedPartial([]byte(`{"min":[1,2],"count":1}`)))
+ err := a.feedPartial([]byte(`{"min":[1,3],"count":1}`))
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go
index 1cf3cadcdb..63995a1a70 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go
@@ -10,10 +10,32 @@ import (
// jsonType enumerates the Cosmos item-ordering classes in ascending order:
//
-// undefined < null < bool < number < string
+// undefined < null < bool < number < string < array < object
//
-// Arrays and objects are not orderable in the Cosmos item-comparison rules;
-// callers that encounter them should treat the result as an error.
+// That is the order the service itself returns mixed types in, and the class
+// ranking is reproducible. The order *within* the array and object classes is
+// not: it is the index's own encoding, and a real container answers
+// [1,2] < [2] < [1,3] and {"b":1} < {"a":2} < {"a":1}, which no function of the
+// values gives back.
+//
+// So two unequal values of one of those classes have no order here, and the
+// two comparators below part ways over them. compareItems fails, for the
+// callers whose answer is the order itself: an ORDER BY that answered from a
+// comparator disagreeing with the per-partition streams it merges would put
+// out rows in an order that is neither the service's nor its own, and under a
+// TOP it would take the wrong rows outright; MIN and MAX fold their partials
+// through the same comparison and would pick an arbitrary one of two arrays.
+// compareItemsTotal instead orders them by canonical JSON, which is
+// repeatable but not the service's, so a caller may use it only where that
+// cannot reach the answer:
+//
+// - a GROUP BY the user did not order, whose rows have no order to be wrong
+// about;
+// - the ORDER BY merge heap, which needs *a* minimum to pop at all. Its
+// order is provisional, and ambiguousFirst then asks the question that
+// matters of the row about to go out — so changing the heap to
+// compareItems would not make it stricter, only fail rows the answer
+// never depended on.
type jsonType int
const (
@@ -22,7 +44,8 @@ const (
jsonBool
jsonNumber
jsonString
- jsonUnorderable // arrays, objects — reject rather than compare
+ jsonArray
+ jsonObject
)
// detectJSONType inspects raw JSON (possibly leading whitespace) and returns
@@ -39,8 +62,10 @@ func detectJSONType(raw json.RawMessage) jsonType {
return jsonBool
case '"':
return jsonString
- case '[', '{':
- return jsonUnorderable
+ case '[':
+ return jsonArray
+ case '{':
+ return jsonObject
}
// digit, minus, plus, or dot → number
if (trimmed[0] >= '0' && trimmed[0] <= '9') || trimmed[0] == '-' || trimmed[0] == '+' || trimmed[0] == '.' {
@@ -49,14 +74,16 @@ func detectJSONType(raw json.RawMessage) jsonType {
return jsonUndefined
}
-// compareItems compares two JSON-encoded scalars using Cosmos item ordering.
+// compareItems compares two JSON-encoded values using Cosmos item ordering.
// Return values follow the usual convention: -1, 0, +1.
-// Returns an error when either side is unorderable (array/object).
+//
+// Values of different classes always compare, and so do two equal values of
+// any class. Two *unequal* arrays, or two unequal objects, do not: the service
+// orders those by its index's own encoding, which no function of the values
+// reproduces, and the error says so by wrapping ErrUnsupportedPlanFeature.
+// Callers that may order such a pair take compareItemsTotal instead.
func compareItems(a, b json.RawMessage) (int, error) {
ta, tb := detectJSONType(a), detectJSONType(b)
- if ta == jsonUnorderable || tb == jsonUnorderable {
- return 0, errors.New("gonative: cannot compare unorderable values (arrays/objects)")
- }
if ta != tb {
if ta < tb {
return -1, nil
@@ -109,7 +136,69 @@ func compareItems(a, b json.RawMessage) (int, error) {
return 0, errors.Wrap(err, "gonative: cmp string b")
}
return strings.Compare(av, bv), nil
+ case jsonArray, jsonObject:
+ order, err := compareComposites(a, b)
+ if err != nil {
+ return 0, err
+ }
+ if order == 0 {
+ return 0, nil
+ }
+ // Equal composites are one value and may be ordered together; unequal
+ // ones the service separates by an encoding this cannot see.
+ return 0, errors.Wrapf(ErrUnsupportedPlanFeature,
+ "gonative: cannot order two different %ss", ta)
default:
return 0, errors.Errorf("gonative: unknown jsonType %d", ta)
}
}
+
+// compareItemsTotal is compareItems extended to a total order: two unequal
+// values of one composite class are ordered by their canonical JSON. That
+// order is not the service's, so a caller either has to be free to pick one —
+// a GROUP BY nobody ordered, or a sort key appended to settle ties — or has to
+// check the rows it hands back, as the merge heap does through
+// ambiguousFirst. See jsonType.
+func compareItemsTotal(a, b json.RawMessage) (int, error) {
+ ta, tb := detectJSONType(a), detectJSONType(b)
+ if ta == tb && (ta == jsonArray || ta == jsonObject) {
+ return compareComposites(a, b)
+ }
+ return compareItems(a, b)
+}
+
+// compareComposites orders two arrays or two objects by their canonical JSON,
+// which is 0 exactly when they are the same value.
+func compareComposites(a, b json.RawMessage) (int, error) {
+ ca, err := canonicalJSON(a)
+ if err != nil {
+ return 0, errors.Wrap(err, "gonative: cmp composite a")
+ }
+ cb, err := canonicalJSON(b)
+ if err != nil {
+ return 0, errors.Wrap(err, "gonative: cmp composite b")
+ }
+ return bytes.Compare(ca, cb), nil
+}
+
+// String names the class in an error a user may read.
+func (t jsonType) String() string {
+ switch t {
+ case jsonUndefined:
+ return "undefined"
+ case jsonNull:
+ return "null"
+ case jsonBool:
+ return "boolean"
+ case jsonNumber:
+ return "number"
+ case jsonString:
+ return "string"
+ case jsonArray:
+ return "array"
+ case jsonObject:
+ return "object"
+ default:
+ return "value"
+ }
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go
index 36248c3fc2..6f9308b8da 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go
@@ -2,6 +2,7 @@ package gonative
import (
"encoding/json"
+ "slices"
"testing"
"github.com/stretchr/testify/assert"
@@ -55,11 +56,85 @@ func TestCompareItems_WithinType(t *testing.T) {
}
}
-func TestCompareItems_UnorderableFails(t *testing.T) {
- _, err := compareItems(json.RawMessage(`[1,2]`), json.RawMessage(`3`))
- require.Error(t, err)
- _, err = compareItems(json.RawMessage(`{}`), json.RawMessage(`null`))
- require.Error(t, err)
+// The service sorts mixed types undefined < null < bool < number < string <
+// array < object; measured on a container holding one row of each.
+func TestCompareItems_RanksCompositesLast(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ a, b json.RawMessage
+ wantCmp int
+ }{
+ {"array after number", json.RawMessage(`[1,2]`), json.RawMessage(`3`), 1},
+ {"array after string", json.RawMessage(`[1,2]`), json.RawMessage(`"zzz"`), 1},
+ {"object after array", json.RawMessage(`{"a":1}`), json.RawMessage(`[9,9]`), 1},
+ {"object after null", json.RawMessage(`{}`), json.RawMessage(`null`), 1},
+ {"array before object", json.RawMessage(`[]`), json.RawMessage(`{}`), -1},
+ {"equal arrays", json.RawMessage(`[1,2]`), json.RawMessage(`[1,2]`), 0},
+ {"equal objects, keys reordered", json.RawMessage(`{"a":1,"b":2}`), json.RawMessage(`{"b":2,"a":1}`), 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := compareItems(tc.a, tc.b)
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantCmp, got)
+ back, err := compareItems(tc.b, tc.a)
+ require.NoError(t, err)
+ assert.Equal(t, -tc.wantCmp, back, "comparison must be antisymmetric")
+ })
+ }
+}
+
+// Two different arrays, or two different objects, have no order the values can
+// give back: an ORDER BY merge that picked one would disagree with the streams
+// it merges, and under a TOP it would return the wrong rows.
+func TestCompareItems_RefusesTwoDifferentComposites(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ a, b json.RawMessage
+ }{
+ {"two arrays", json.RawMessage(`[1,2]`), json.RawMessage(`[1,3]`)},
+ {"two objects", json.RawMessage(`{"a":1}`), json.RawMessage(`{"b":1}`)},
+ {"nested", json.RawMessage(`[[1]]`), json.RawMessage(`[[2]]`)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := compareItems(tc.a, tc.b)
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+ _, err = compareItems(tc.b, tc.a)
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+ })
+ }
+}
+
+// compareItemsTotal is the one for a caller free to pick an order: it must be
+// total and the same on every run.
+func TestCompareItemsTotal_OrdersCompositesConsistently(t *testing.T) {
+ values := []json.RawMessage{
+ json.RawMessage(`[1,3]`), json.RawMessage(`[2]`), json.RawMessage(`[1,2]`),
+ json.RawMessage(`{"b":1}`), json.RawMessage(`{"a":2}`), json.RawMessage(`{"a":1}`),
+ }
+ sortOnce := func() []string {
+ shuffled := slices.Clone(values)
+ slices.SortStableFunc(shuffled, func(a, b json.RawMessage) int {
+ order, err := compareItemsTotal(a, b)
+ require.NoError(t, err)
+ return order
+ })
+ out := make([]string, 0, len(shuffled))
+ for _, v := range shuffled {
+ out = append(out, string(v))
+ }
+ return out
+ }
+ first := sortOnce()
+ assert.Equal(t, first, sortOnce())
+ // Whatever the order within each class, every array precedes every object.
+ for i, v := range first {
+ if v[0] == '{' {
+ for _, rest := range first[i:] {
+ assert.Equal(t, byte('{'), rest[0], "an array sorted after an object: %v", first)
+ }
+ break
+ }
+ }
}
func TestCompareItems_HandlesLeadingWhitespace(t *testing.T) {
@@ -81,8 +156,8 @@ func TestDetectJSONType(t *testing.T) {
{`42`, jsonNumber},
{`-1.5`, jsonNumber},
{`"x"`, jsonString},
- {`[1,2]`, jsonUnorderable},
- {`{"a":1}`, jsonUnorderable},
+ {`[1,2]`, jsonArray},
+ {`{"a":1}`, jsonObject},
{` "pad" `, jsonString},
}
for _, tc := range cases {
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go b/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go
index 5cef1119ab..266605bf17 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go
@@ -4,6 +4,7 @@
// The CosmosDB driver engages it per query: the gateway runs what it can on
// its own, and only a cross-partition query it refuses is replayed through
// this engine. SupportedFeatures lists the operators the engine implements;
-// a plan that needs anything else fails with ErrUnsupportedPlanFeature, and
-// the caller then surfaces the gateway's original error.
+// a plan that needs anything else fails with ErrUnsupportedPlanFeature before
+// a row is read, and the caller then surfaces the gateway's original error. A
+// pipeline already running can fail with it too — see that sentinel.
package gonative
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
index e13f1ea13b..43c3dbecdd 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go
@@ -1,8 +1,10 @@
package gonative
import (
+ "bytes"
"encoding/json"
"slices"
+ "strings"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
"github.com/pkg/errors"
@@ -78,16 +80,9 @@ type groupPipeline struct {
// is what the output falls back to when the keys cannot be ordered.
order []string
- // descending says which way to order the output rows, and
- // directionKnown whether the data has told us yet. A GROUP BY the user
- // also ordered
- // reaches the engine as a plan with no order fields at all — the ORDER BY
- // survives only inside rewrittenQuery — so the direction can be read only
- // off the responses: the gateway returns each partition's own groups in
- // the requested order, and the first adjacent pair that differs says
- // which way they run.
- descending bool
- directionKnown bool
+ // sort is how to order the output rows: which group key to compare
+ // first, which way, and how strictly — see groupSortKeys.
+ sort []groupSortKey
// pending holds fully finalized rows awaiting the caller's next Run().
pending [][]byte
@@ -130,6 +125,11 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error
}
}
+ sortKeys, err := groupSortKeys(qi)
+ if err != nil {
+ return nil, err
+ }
+
return &groupPipeline{
// GROUP BY plans always carry a non-empty rewrittenQuery that wraps the
// user's projection into {groupByItems, payload}; running the original
@@ -139,6 +139,7 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error
aliases: aliases,
kinds: kinds,
groups: make(map[string]*groupState),
+ sort: sortKeys,
}, nil
}
@@ -193,7 +194,6 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {
if err != nil {
return err
}
- p.noteOrdering(resp.Documents)
for docIdx, doc := range resp.Documents {
if len(doc.GroupByItems) == 0 {
return errors.Errorf("gonative: group doc %d missing groupByItems", docIdx)
@@ -260,7 +260,9 @@ func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {
// sortGroups. Walking p.groups instead would emit them in Go's randomized map
// order, which made the same query answer in a different order every run.
func (p *groupPipeline) finalize() error {
- p.sortGroups()
+ if err := p.sortGroups(); err != nil {
+ return err
+ }
for _, key := range p.order {
row, err := p.buildRow(p.groups[key])
if err != nil {
@@ -272,70 +274,231 @@ func (p *groupPipeline) finalize() error {
return nil
}
-// noteOrdering reads the output direction off one partition's response, which
-// is the only place a GROUP BY's ORDER BY reaches the engine. Gives up on a
-// key it cannot compare: sortGroups checks for those itself.
-func (p *groupPipeline) noteOrdering(docs []groupDoc) {
- if p.directionKnown {
- return
+// orderByKeyword is how the gateway spells the clause in a rewritten query.
+const orderByKeyword = "ORDER BY"
+
+// orderByKey is one key of an ORDER BY clause.
+type orderByKey struct {
+ expr string
+ descending bool
+}
+
+// groupSortKey names one group key to order the output rows by: its position
+// in groupByItems, which way to compare it, and whether the query named it.
+//
+// A key the query named has to be compared the service's way or not at all. A
+// key appended to settle ties the clause says nothing about takes any
+// repeatable order, so it is compared with compareItemsTotal and never fails.
+type groupSortKey struct {
+ index int
+ descending bool
+ named bool
+}
+
+// groupSortKeys works out how to order a GROUP BY's output rows. A GROUP BY
+// the user also ordered reaches the engine as a plan with no order fields at
+// all — orderBy and orderByExpressions stay empty and the ORDER BY survives
+// only inside the rewritten query — so the clause is read from that text.
+//
+// The engine orders these rows by comparing their group keys, so every key the
+// clause names must be one of the GROUP BY expressions. An order it cannot
+// produce that way — by an aggregate, or by a column the query does not group
+// by — fails the query rather than answering in an order nobody asked for. The
+// gateway refuses those queries itself today, so this is the engine holding the
+// line rather than a reachable error.
+//
+// Group keys the clause leaves out are appended ascending. The clause does not
+// constrain them, and ordering them too is what makes two runs of one query
+// answer alike.
+func groupSortKeys(qi planQueryInfo) ([]groupSortKey, error) {
+ keys, found := trailingOrderBy(qi.RewrittenQuery)
+ if found && len(keys) == 0 {
+ return nil, errors.Wrap(ErrUnsupportedPlanFeature, "ORDER BY is not a plain list of keys")
}
- for i := 1; i < len(docs); i++ {
- order, err := compareGroupKeys(docs[i-1].GroupByItems, docs[i].GroupByItems)
- if err != nil {
- return
+ sortKeys := make([]groupSortKey, 0, len(qi.GroupByExpressions))
+ named := make([]bool, len(qi.GroupByExpressions))
+ for _, key := range keys {
+ index := slices.Index(qi.GroupByExpressions, key.expr)
+ if index < 0 {
+ return nil, errors.Wrapf(ErrUnsupportedPlanFeature,
+ "ORDER BY key %q is not one of the GROUP BY expressions", key.expr)
}
- if order != 0 {
- p.descending = order > 0
- p.directionKnown = true
- return
+ if named[index] {
+ continue
+ }
+ named[index] = true
+ sortKeys = append(sortKeys, groupSortKey{index: index, descending: key.descending, named: true})
+ }
+ for index, taken := range named {
+ if !taken {
+ sortKeys = append(sortKeys, groupSortKey{index: index})
}
}
+ return sortKeys, nil
}
-// sortGroups puts p.order in group-key order, the direction taken from the
-// gateway's own per-partition streams. A GROUP BY without an ORDER BY has no
-// defined output order, so ordering those rows too costs nothing and is what
-// makes every run answer the same way.
+// trailingOrderBy splits the ORDER BY clause that ends query into its keys.
+// found is false when the query has no trailing ORDER BY; keys is nil with
+// found true when the clause is not the plain `<expression> [ASC|DESC]` list
+// the gateway writes there, which the caller must treat as an order it cannot
+// honour rather than as no order at all.
//
-// Leaves first-seen order in place when any group key is a shape Cosmos
-// cannot order (an array or an object), since the comparator would have to
-// invent an answer for those.
-func (p *groupPipeline) sortGroups() {
- for _, key := range p.order {
- for _, item := range p.groups[key].items {
- if detectJSONType(item.Item) == jsonUnorderable {
- return
+// The words are matched outside quoted text, so a property path that needs
+// bracket notation — c["order by"] — is a key and not a clause, whichever it
+// is spelled as.
+//
+// The keys are plain property paths because that is all the service allows a
+// grouped query to order by: `GROUP BY SUBSTRING(c.x, 0, 1) ORDER BY
+// SUBSTRING(c.x, 0, 1)` is refused outright with SC2103, "not contained in
+// either an aggregate function or the GROUP BY clause", however exactly the
+// two spellings match. So a key here holds no comma and no bracketed
+// arithmetic of its own.
+func trailingOrderBy(query string) (keys []orderByKey, found bool) {
+ at := lastUnquotedIndex(query, orderByKeyword)
+ if at < 0 {
+ return nil, false
+ }
+ tail := query[at+len(orderByKeyword):]
+ for _, item := range splitUnquoted(tail, ',') {
+ fields := splitUnquoted(item, ' ', '\t', '\n', '\r')
+ var key orderByKey
+ switch len(fields) {
+ case 1:
+ key.expr = fields[0]
+ case 2:
+ key.expr = fields[0]
+ switch strings.ToUpper(fields[1]) {
+ case "ASC":
+ case "DESC":
+ key.descending = true
+ default:
+ return nil, true
}
+ default:
+ return nil, true
}
+ keys = append(keys, key)
}
- slices.SortStableFunc(p.order, func(a, b string) int {
- order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items)
- if err != nil {
- // Unreachable: every key was checked orderable just above.
- return 0
+ if len(keys) == 0 {
+ return nil, true
+ }
+ return keys, true
+}
+
+// lastUnquotedIndex returns the last index at which needle occurs in s outside
+// quoted text, ignoring case, or -1.
+func lastUnquotedIndex(s, needle string) int {
+ quoted := quotedBytes(s)
+ for i := len(s) - len(needle); i >= 0; i-- {
+ if !quoted[i] && strings.EqualFold(s[i:i+len(needle)], needle) {
+ return i
+ }
+ }
+ return -1
+}
+
+// splitUnquoted splits s on any of seps that sits outside quoted text,
+// dropping empty pieces so runs of whitespace behave like one separator.
+func splitUnquoted(s string, seps ...byte) []string {
+ quoted := quotedBytes(s)
+ var out []string
+ start := -1
+ for i := range len(s) {
+ if !quoted[i] && slices.Contains(seps, s[i]) {
+ if start >= 0 {
+ out = append(out, s[start:i])
+ start = -1
+ }
+ continue
+ }
+ if start < 0 {
+ start = i
+ }
+ }
+ if start >= 0 {
+ out = append(out, s[start:])
+ }
+ return out
+}
+
+// quotedBytes marks the bytes of s that sit inside quoted text. A rewritten
+// query quotes a property path spelled in bracket notation and any string
+// literal the query compares against, and the words of a clause mean nothing
+// inside either.
+func quotedBytes(s string) []bool {
+ quoted := make([]bool, len(s))
+ var quote byte
+ for i := 0; i < len(s); i++ {
+ c := s[i]
+ if quote == 0 {
+ if c == '"' || c == '\'' {
+ quote = c
+ quoted[i] = true
+ }
+ continue
+ }
+ quoted[i] = true
+ if c == '\\' && i+1 < len(s) {
+ // An escaped quote does not close the span, and the byte it
+ // escapes is inside it either way.
+ i++
+ quoted[i] = true
+ continue
+ }
+ if c == quote {
+ quote = 0
}
- if p.descending {
- return -order
+ }
+ return quoted
+}
+
+// sortGroups puts p.order in group-key order, in the direction the query asked
+// for. Each key carries how strictly it is compared — see groupSortKey — so a
+// key the query named fails rather than take an invented order, while one
+// appended to settle ties takes any repeatable one.
+func (p *groupPipeline) sortGroups() error {
+ var failed error
+ slices.SortStableFunc(p.order, func(a, b string) int {
+ order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items, p.sort)
+ if err != nil && failed == nil {
+ failed = err
}
return order
})
+ return failed
}
-// compareGroupKeys orders two group keys under Cosmos item ordering, column
-// by column.
-func compareGroupKeys(a, b []groupByItem) (int, error) {
- for i := range min(len(a), len(b)) {
- order, err := compareItems(a[i].Item, b[i].Item)
+// compareGroupKeys orders two group keys under Cosmos item ordering, comparing
+// the columns sortKeys names in the order, direction and strictness it gives.
+func compareGroupKeys(a, b []groupByItem, sortKeys []groupSortKey) (int, error) {
+ for _, key := range sortKeys {
+ if key.index >= len(a) || key.index >= len(b) {
+ continue
+ }
+ cmp := compareItemsTotal
+ if key.named {
+ cmp = compareItems
+ }
+ order, err := cmp(a[key.index].Item, b[key.index].Item)
if err != nil {
return 0, err
}
if order != 0 {
+ if key.descending {
+ return -order, nil
+ }
return order, nil
}
}
return len(a) - len(b), nil
}
+// isUndefinedItem reports whether a groupByItems entry carries no value,
+// which is how the gateway spells a key the document does not define.
+func isUndefinedItem(raw json.RawMessage) bool {
+ return len(bytes.TrimLeft(raw, " \t\n\r")) == 0
+}
+
// buildRow assembles the final JSON object for a single group, preserving
// the alias order from groupByAliases. JSON object key order in Go's encoder
// depends on sort for map marshaling, so we write the object manually.
@@ -347,11 +510,15 @@ func (p *groupPipeline) buildRow(s *groupState) ([]byte, error) {
var buf []byte
buf = append(buf, '{')
for i, alias := range p.aliases {
- value := json.RawMessage("null")
+ var value json.RawMessage
if p.kinds[i] == "" {
- if v, ok := s.keyValues[alias]; ok {
- value = v
+ v, ok := s.keyValues[alias]
+ if !ok {
+ // The group's key is undefined here, and the service answers
+ // that with the alias left out of the row.
+ continue
}
+ value = v
} else {
final, err := s.aggregators[i].finalize()
if err != nil {
@@ -383,13 +550,22 @@ func groupKey(items []groupByItem) (string, error) {
// Build a slice of the inner "item" values, canonicalize each, then join.
// Using a combined key of canonicalized element JSON keeps equality
// deterministic even for complex multi-key GROUP BY.
+ //
+ // A key the documents leave undefined arrives with no "item" at all. It is
+ // a group of its own — the service counts it apart from an explicit null —
+ // so each part is wrapped in an array and an undefined one is the empty
+ // array, which no value's canonical form can collide with.
parts := make([]json.RawMessage, len(items))
for i, it := range items {
+ if isUndefinedItem(it.Item) {
+ parts[i] = json.RawMessage(`[]`)
+ continue
+ }
c, err := canonicalJSON(it.Item)
if err != nil {
return "", errors.Wrapf(err, "gonative: canonicalize group key part %d", i)
}
- parts[i] = c
+ parts[i] = append(append(json.RawMessage{'['}, c...), ']')
}
b, err := json.Marshal(parts)
if err != nil {
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
index bd4ffd12c9..40e952741e 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go
@@ -2,6 +2,8 @@ package gonative
import (
"encoding/json"
+ "fmt"
+ "strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
@@ -41,6 +43,14 @@ func mkGroupBody(t *testing.T, entries ...struct {
return b
}
+// mkGroupBodyRaw builds a partition response from literal document JSON, for
+// shapes mkGroupBody cannot spell — a groupByItems entry with no "item" at
+// all, which is how the gateway reports a key the document leaves undefined.
+func mkGroupBodyRaw(t *testing.T, docs ...string) []byte {
+ t.Helper()
+ return []byte(`{"Documents":[` + strings.Join(docs, ",") + `]}`)
+}
+
type groupEntry = struct {
key, payload string
}
@@ -354,41 +364,259 @@ func countByCountryPlan() *planDoc {
map[string]string{"country": "", "n": "Count"})
}
-// A GROUP BY the user also ordered reaches the engine as a plan with no order
-// fields, so the direction comes off the data: the gateway returns each
-// partition's own groups in the requested order.
-func TestGroupPipeline_EmitsAscendingWhenThePartitionsRunAscending(t *testing.T) {
- p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})
+// orderedCountByCountryPlan is countByCountryPlan for a query the user also
+// ordered. Such a plan carries no order fields at all — the clause survives
+// only in the rewritten query, spelled as the gateway writes it.
+func orderedCountByCountryPlan(clause string) *planDoc {
+ plan := countByCountryPlan()
+ plan.QueryInfo.RewrittenQuery = `SELECT [{"item": c.country}] AS groupByItems, ` +
+ `{"country": c.country, "n": {"item": COUNT(1)}} AS payload` + "\n" +
+ `FROM c` + "\n" + `GROUP BY c.country` + "\n" + clause
+ return plan
+}
+
+// The direction comes from the query, not from the order the ranges happen to
+// answer in: these two feed the same rows and differ only in the clause.
+func TestGroupPipeline_EmitsAscendingWhenTheQueryOrdersAscending(t *testing.T) {
+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country"), []string{"0", "1"})
require.NoError(t, err)
- rows := driveGroup(t, p, map[string][]byte{
+ rows := driveGroup(t, p, descendingCountryFeed(t))
+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))
+}
+
+func TestGroupPipeline_EmitsDescendingWhenTheQueryOrdersDescending(t *testing.T) {
+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country DESC"), []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, descendingCountryFeed(t))
+ assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))
+}
+
+func descendingCountryFeed(t *testing.T) map[string][]byte {
+ t.Helper()
+ return map[string][]byte{
"0": mkGroupBody(t,
- groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},
+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
),
"1": mkGroupBody(t,
- groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},
+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
),
+ }
+}
+
+// A range that reports a single group says nothing about which way the groups
+// run, so the direction has to come from the clause.
+func TestGroupPipeline_EmitsDescendingWhenEveryRangeHasOneGroup(t *testing.T) {
+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country DESC"), []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`}),
+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`}),
})
- assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))
+ assert.Equal(t, []string{"BR", "AE"}, groupKeysOf(t, rows, "country"))
+}
+
+// An order the engine cannot produce by comparing group keys has to fail the
+// query: ordering by the keys anyway would answer in an order the user did not
+// ask for. The service refuses these itself today, so this is the engine
+// holding the line.
+func TestGroupPipeline_RejectsAnOrderItCannotProduce(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ clause string
+ }{
+ {"by an aggregate", "ORDER BY COUNT(1) DESC"},
+ {"by an output alias", "ORDER BY n DESC"},
+ {"by a column the query does not group by", "ORDER BY c.city"},
+ {"by something that is not a list of keys", "ORDER BY c.country NULLS LAST"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := newGroupPipeline(orderedCountByCountryPlan(tc.clause), []string{"0", "1"})
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+ })
+ }
+}
+
+// Cosmos gives every ORDER BY key its own direction, and accepts a clause that
+// names only some of the group keys.
+func TestGroupPipeline_OrdersMultipleKeysEachItsOwnWay(t *testing.T) {
+ feed := map[string][]byte{
+ "0": mkTwoKeyGroupBody(t,
+ twoKeyEntry{keys: []string{`"AE"`, `2`}, payload: `{"country":"AE","city":2,"n":{"item":7}}`},
+ twoKeyEntry{keys: []string{`"AE"`, `1`}, payload: `{"country":"AE","city":1,"n":{"item":8}}`},
+ ),
+ "1": mkTwoKeyGroupBody(t,
+ twoKeyEntry{keys: []string{`"BR"`, `1`}, payload: `{"country":"BR","city":1,"n":{"item":9}}`},
+ twoKeyEntry{keys: []string{`"BR"`, `2`}, payload: `{"country":"BR","city":2,"n":{"item":6}}`},
+ ),
+ }
+
+ for _, tc := range []struct {
+ name string
+ clause string
+ want []string
+ }{
+ {"ascending then descending", "ORDER BY c.country, c.city DESC", []string{"AE/2", "AE/1", "BR/2", "BR/1"}},
+ {"descending then ascending", "ORDER BY c.country DESC, c.city", []string{"BR/1", "BR/2", "AE/1", "AE/2"}},
+ {"only the second key", "ORDER BY c.city DESC", []string{"AE/2", "BR/2", "AE/1", "BR/1"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ p, err := newGroupPipeline(countryCityPlan(tc.clause), []string{"0", "1"})
+ require.NoError(t, err)
+ assert.Equal(t, tc.want, countryCityKeysOf(t, driveGroup(t, p, feed)))
+ })
+ }
}
-func TestGroupPipeline_EmitsDescendingWhenThePartitionsRunDescending(t *testing.T) {
- p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})
+// A group key the clause leaves out still gets an order, so that two runs of
+// one query answer alike however the ranges happened to report their groups.
+func TestGroupPipeline_OrdersKeysTheClauseLeavesOut(t *testing.T) {
+ p, err := newGroupPipeline(countryCityPlan("ORDER BY c.city DESC"), []string{"0", "1"})
require.NoError(t, err)
rows := driveGroup(t, p, map[string][]byte{
- "0": mkGroupBody(t,
- groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},
- groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},
+ "0": mkTwoKeyGroupBody(t,
+ twoKeyEntry{keys: []string{`"BR"`, `2`}, payload: `{"country":"BR","city":2,"n":{"item":6}}`},
),
- "1": mkGroupBody(t,
- groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},
- groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},
+ "1": mkTwoKeyGroupBody(t,
+ twoKeyEntry{keys: []string{`"AE"`, `2`}, payload: `{"country":"AE","city":2,"n":{"item":7}}`},
),
})
- assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))
+ assert.Equal(t, []string{"AE/2", "BR/2"}, countryCityKeysOf(t, rows))
+}
+
+// countryCityPlan groups by two keys, which the gateway reports in
+// groupByItems in the order groupByExpressions names them.
+func countryCityPlan(clause string) *planDoc {
+ plan := newGroupPlan("c.country",
+ []string{"country", "city", "n"},
+ map[string]string{"country": "", "city": "", "n": "Count"})
+ plan.QueryInfo.GroupByExpressions = []string{"c.country", "c.city"}
+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.country, c.city\n" + clause
+ return plan
+}
+
+// mkTwoKeyGroupBody is mkGroupBody for a two-expression GROUP BY: each
+// document's groupByItems carries both keys, in groupByExpressions order.
+func mkTwoKeyGroupBody(t *testing.T, entries ...struct {
+ keys []string
+ payload string
+}) []byte {
+ t.Helper()
+ docs := make([]map[string]any, 0, len(entries))
+ for _, e := range entries {
+ var payloadObj map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal([]byte(e.payload), &payloadObj), "invalid payload fixture: %s", e.payload)
+ items := make([]map[string]any, 0, len(e.keys))
+ for _, key := range e.keys {
+ items = append(items, map[string]any{"item": json.RawMessage(key)})
+ }
+ docs = append(docs, map[string]any{"groupByItems": items, "payload": payloadObj})
+ }
+ b, err := json.Marshal(map[string]any{"Documents": docs})
+ require.NoError(t, err)
+ return b
+}
+
+type twoKeyEntry = struct {
+ keys []string
+ payload string
+}
+
+func countryCityKeysOf(t *testing.T, rows [][]byte) []string {
+ t.Helper()
+ out := make([]string, 0, len(rows))
+ for _, r := range rows {
+ var obj struct {
+ Country string `json:"country"`
+ City int `json:"city"`
+ }
+ require.NoError(t, json.Unmarshal(r, &obj), "row: %s", r)
+ out = append(out, fmt.Sprintf("%s/%d", obj.Country, obj.City))
+ }
+ return out
+}
+
+// A key whose name holds a clause keyword is still a key.
+func TestGroupPipeline_ReadsAnOrderWhoseKeyNameHoldsAKeyword(t *testing.T) {
+ plan := newGroupPlan("c.nowhere", []string{"nowhere", "n"}, map[string]string{"nowhere": "", "n": "Count"})
+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.nowhere\nORDER BY c.nowhere DESC"
+
+ p, err := newGroupPipeline(plan, []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"nowhere":"AE","n":{"item":2}}`}),
+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"nowhere":"BR","n":{"item":4}}`}),
+ })
+ assert.Equal(t, []string{"BR", "AE"}, groupKeysOf(t, rows, "nowhere"))
+}
+
+// A property path that needs bracket notation is a key like any other, even
+// when the property is named after the clause itself. Both plans here are what
+// the service answered for `GROUP BY c["country-name"]` and
+// `GROUP BY c["order by"]`.
+func TestGroupPipeline_ReadsAnOrderOnABracketPath(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ expr string
+ rest string
+ want []string
+ }{
+ {
+ name: "a path that has to be quoted",
+ expr: `c["country-name"]`,
+ rest: "\nORDER BY c[\"country-name\"] DESC",
+ want: []string{"BR", "AE"},
+ },
+ {
+ name: "a property named after the clause",
+ expr: `c["order by"]`,
+ rest: "\nORDER BY c[\"order by\"] DESC",
+ want: []string{"BR", "AE"},
+ },
+ {
+ name: "a property named after the clause, unordered",
+ expr: `c["order by"]`,
+ rest: "",
+ want: []string{"AE", "BR"},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ plan := newGroupPlan(tc.expr, []string{"k", "n"}, map[string]string{"k": "", "n": "Count"})
+ plan.QueryInfo.GroupByExpressions = []string{tc.expr}
+ plan.QueryInfo.RewrittenQuery = "SELECT [{\"item\": " + tc.expr + "}] AS groupByItems, " +
+ "{\"k\": " + tc.expr + ", \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY " + tc.expr + tc.rest
+
+ p, err := newGroupPipeline(plan, []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"k":"AE","n":{"item":2}}`}),
+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"k":"BR","n":{"item":4}}`}),
+ })
+ assert.Equal(t, tc.want, groupKeysOf(t, rows, "k"))
+ })
+ }
+}
+
+// The words can appear in the query body — in a string literal the user wrote
+// — without the query naming an order.
+func TestGroupPipeline_IgnoresOrderByInsideTheQueryBody(t *testing.T) {
+ plan := countByCountryPlan()
+ plan.QueryInfo.RewrittenQuery = `SELECT [{"item": c.country}] AS groupByItems, ` +
+ `{"country": c.country, "n": {"item": COUNT(1)}} AS payload` + "\n" +
+ `FROM c` + "\n" + `WHERE c.note != 'order by me'` + "\n" + `GROUP BY c.country`
+
+ p, err := newGroupPipeline(plan, []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, descendingCountryFeed(t))
+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))
}
// Without an ORDER BY the output order is unspecified, but it still has to be
@@ -417,9 +645,10 @@ func TestGroupPipeline_OutputOrderIsStableAcrossRuns(t *testing.T) {
}
}
-// A group key Cosmos cannot order — an array or an object — leaves the output
-// in first-seen order rather than asking the comparator to invent one.
-func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {
+// A group key the service cannot order by value — an array or an object — is
+// still ordered here, by its canonical JSON, so the same query does not answer
+// in a different order on the next run.
+func TestGroupPipeline_OrdersCompositeKeysConsistently(t *testing.T) {
plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})
p, err := newGroupPipeline(plan, []string{"0"})
require.NoError(t, err)
@@ -431,8 +660,109 @@ func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {
),
})
require.Len(t, rows, 2)
- assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[0]))
- assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[1]))
+ assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[0]))
+ assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[1]))
+}
+
+// An ORDER BY on a group key the engine cannot order — two different arrays —
+// fails the query. Ordering the groups by anything else would answer in an
+// order the user did not ask for, which is the whole point of reading the
+// clause in the first place.
+func TestGroupPipeline_RefusesAnOrderedKeyItCannotOrder(t *testing.T) {
+ plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})
+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.tags\nORDER BY c.tags"
+ p, err := newGroupPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ res, err := p.Run()
+ require.NoError(t, err)
+ require.NotEmpty(t, res.Requests)
+ body := mkGroupBody(t,
+ groupEntry{key: `["b"]`, payload: `{"tags":["b"],"n":{"item":1}}`},
+ groupEntry{key: `["a"]`, payload: `{"tags":["a"],"n":{"item":2}}`},
+ )
+ results := make([]queryengine.QueryResult, 0, len(res.Requests))
+ for _, req := range res.Requests {
+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, body, ""))
+ }
+ require.NoError(t, p.ProvideData(results))
+
+ _, err = p.Run()
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+}
+
+// A clause that names one of two group keys leaves the other to settle ties.
+// The query asked nothing of that key, so either tie order satisfies it and an
+// array there must not fail the query — only a key the clause named has to be
+// ordered the service's way.
+func TestGroupPipeline_SettlesTiesOnAKeyTheClauseOmits(t *testing.T) {
+ plan := newGroupPlan("c.country", []string{"country", "tags", "n"},
+ map[string]string{"country": "", "tags": "", "n": "Count"})
+ plan.QueryInfo.GroupByExpressions = []string{"c.country", "c.tags"}
+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.country, c.tags\nORDER BY c.country DESC"
+
+ p, err := newGroupPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkTwoKeyGroupBody(t,
+ twoKeyEntry{keys: []string{`"AE"`, `["b"]`}, payload: `{"country":"AE","tags":["b"],"n":{"item":1}}`},
+ twoKeyEntry{keys: []string{`"BR"`, `["z"]`}, payload: `{"country":"BR","tags":["z"],"n":{"item":2}}`},
+ twoKeyEntry{keys: []string{`"AE"`, `["a"]`}, payload: `{"country":"AE","tags":["a"],"n":{"item":3}}`},
+ ),
+ })
+ got := make([]string, 0, len(rows))
+ for _, row := range rows {
+ var object struct {
+ Country string `json:"country"`
+ Tags []string `json:"tags"`
+ }
+ require.NoError(t, json.Unmarshal(row, &object), "row: %s", row)
+ got = append(got, object.Country+"/"+strings.Join(object.Tags, ","))
+ }
+ assert.Equal(t, []string{"BR/z", "AE/a", "AE/b"}, got)
+}
+
+// A column the documents do not all define is ordinary in a schemaless store.
+// The service answers it as a group of its own, counted apart from an explicit
+// null and spelled with the alias left out of the row, and sends that group's
+// key as a groupByItems entry with no "item" at all — the shape this fixture
+// exists for.
+func TestGroupPipeline_GroupsAnUndefinedKeyOnItsOwn(t *testing.T) {
+ plan := newGroupPlan("c.v", []string{"k", "n"}, map[string]string{"k": "", "n": "Count"})
+ p, err := newGroupPipeline(plan, []string{"0", "1"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBodyRaw(t,
+ `{"groupByItems":[{}],"payload":{"n":{"item":2}}}`,
+ `{"groupByItems":[{"item":null}],"payload":{"k":null,"n":{"item":1}}}`),
+ "1": mkGroupBodyRaw(t,
+ `{"groupByItems":[{"item":"a"}],"payload":{"k":"a","n":{"item":3}}}`,
+ `{"groupByItems":[{}],"payload":{"n":{"item":5}}}`),
+ })
+ require.Len(t, rows, 3)
+ assert.JSONEq(t, `{"n":7}`, string(rows[0]), "the undefined group merges across ranges and omits the alias")
+ assert.JSONEq(t, `{"k":null,"n":1}`, string(rows[1]), "an explicit null is its own group")
+ assert.JSONEq(t, `{"k":"a","n":3}`, string(rows[2]))
+}
+
+// A multi-key GROUP BY where one key is undefined in some rows: the undefined
+// slot still distinguishes the group.
+func TestGroupPipeline_GroupsOnAPartlyUndefinedKey(t *testing.T) {
+ plan := newGroupPlan("c.a", []string{"a", "b", "n"}, map[string]string{"a": "", "b": "", "n": "Count"})
+ plan.QueryInfo.GroupByExpressions = []string{"c.a", "c.b"}
+ p, err := newGroupPipeline(plan, []string{"0"})
+ require.NoError(t, err)
+
+ rows := driveGroup(t, p, map[string][]byte{
+ "0": mkGroupBodyRaw(t,
+ `{"groupByItems":[{"item":"x"},{}],"payload":{"a":"x","n":{"item":4}}}`,
+ `{"groupByItems":[{"item":"x"},{"item":7}],"payload":{"a":"x","b":7,"n":{"item":9}}}`),
+ })
+ require.Len(t, rows, 2)
+ assert.JSONEq(t, `{"a":"x","n":4}`, string(rows[0]))
+ assert.JSONEq(t, `{"a":"x","b":7,"n":9}`, string(rows[1]))
}
// MIN over a group where the column is undefined in every row has no value,
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
index 5203d9af7e..6c81b4b397 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go
@@ -70,8 +70,9 @@ func (s *partitionStream) pop() *orderByDoc {
}
// mergeHeap implements heap.Interface over partition indices. The comparator
-// uses compareItems on each partition's current head's orderByItems[0], with
-// the sort direction flipped for descending ORDER BY.
+// uses compareItemsTotal on each partition's current head's orderByItems[0],
+// with the sort direction flipped for descending ORDER BY — see Less for why
+// the total comparator rather than the strict one.
type mergeHeap struct {
partitions []*partitionStream
idx []int // heap of partition indices
@@ -85,11 +86,16 @@ func (h *mergeHeap) Less(i, j int) bool {
// partitions from the heap before they can be compared.
a, _ := h.partitions[h.idx[i]].peek()
b, _ := h.partitions[h.idx[j]].peek()
- cmp, err := compareItems(a.OrderByItems[0].Item, b.OrderByItems[0].Item)
+ // The heap orders with compareItemsTotal so that it always has a minimum,
+ // and because that order follows the service's class ranking the minimum's
+ // class is the right one. Which of two values inside the array or object
+ // class comes first the service alone knows, so merge asks that question
+ // of the row it is about to emit — see ambiguousFirst — rather than of
+ // every pair the heap happens to sift.
+ cmp, err := compareItemsTotal(a.OrderByItems[0].Item, b.OrderByItems[0].Item)
if err != nil {
- // compareItems rejects unorderable shapes (arrays, objects). For
- // ORDER BY scalars this should never happen; if it does, fall back to
- // stable-by-partition-index order so the heap remains well-defined.
+ // The bytes are not JSON; keep the heap well-defined and let the row
+ // fail when it is emitted.
return h.idx[i] < h.idx[j]
}
if h.descending {
@@ -217,11 +223,11 @@ func (p *orderByPipeline) IsComplete() bool {
if len(p.pending) > 0 {
return false
}
- // LIMIT reached — the caller has everything it asked for. Any rows still
- // sitting in partition streams were produced speculatively by the gateway
- // (it fetched OFFSET+LIMIT rows per partition to cover any interleaving)
- // and are safe to drop.
- if p.limit >= 0 && p.emitted >= p.limit {
+ // The caller has everything it asked for. Any rows still sitting in
+ // partition streams were produced speculatively by the gateway (it fetched
+ // OFFSET+LIMIT rows per partition to cover any interleaving) and are safe
+ // to drop.
+ if p.windowFull() {
return true
}
if !p.fanout.allDrained() {
@@ -322,6 +328,13 @@ func (p *orderByPipeline) merge() error {
if p.waitingForPage() {
return nil
}
+ // A window already satisfied ends the merge before it touches a row: a
+ // page that landed after the last emission still holds rows the result
+ // does not include, and merging them would hand one to the caller or ask
+ // an order of them that nobody needs.
+ if p.windowFull() {
+ return nil
+ }
h := &mergeHeap{partitions: p.partitions, descending: p.descending}
for i, s := range p.partitions {
if _, ok := s.peek(); ok {
@@ -331,10 +344,6 @@ func (p *orderByPipeline) merge() error {
heap.Init(h)
for h.Len() > 0 {
- if p.limit >= 0 && p.emitted >= p.limit {
- // LIMIT reached — the caller has everything it asked for.
- return nil
- }
top, ok := heap.Pop(h).(int)
if !ok {
// Unreachable: the heap is seeded with partition indexes only.
@@ -342,6 +351,9 @@ func (p *orderByPipeline) merge() error {
}
s := p.partitions[top]
doc := s.pop()
+ if err := p.ambiguousFirst(doc, top); err != nil {
+ return err
+ }
fresh, err := p.firstSighting(doc.Payload)
if err != nil {
return err
@@ -356,6 +368,12 @@ func (p *orderByPipeline) merge() error {
p.pending = append(p.pending, append([]byte(nil), doc.Payload...))
p.emitted++
}
+ if p.windowFull() {
+ // The caller has everything it asked for. Putting the range back
+ // would compare its next row against the other heads, and a row
+ // outside the window has no business failing the query.
+ return nil
+ }
if _, ok := s.peek(); ok {
heap.Push(h, top)
continue
@@ -367,6 +385,54 @@ func (p *orderByPipeline) merge() error {
return nil
}
+// ambiguousFirst reports an error when another range's head ties with doc for
+// the position doc is about to take, in a way the engine cannot settle: the
+// same composite class, a different value. The service orders within those
+// classes by its index's own encoding, so which of the two comes first is not
+// knowable here.
+//
+// Every scalar is unambiguous, and so is a composite no other head shares a
+// class with — including one the heap reached only because the ranges below it
+// drained. A later row of the same range is not a competitor: the service
+// already placed it behind doc in that range's own stream.
+func (p *orderByPipeline) ambiguousFirst(doc *orderByDoc, from int) error {
+ if len(doc.OrderByItems) == 0 {
+ return nil
+ }
+ item := doc.OrderByItems[0].Item
+ class := detectJSONType(item)
+ if class != jsonArray && class != jsonObject {
+ return nil
+ }
+ for i, stream := range p.partitions {
+ if i == from {
+ continue
+ }
+ head, ok := stream.peek()
+ if !ok || len(head.OrderByItems) == 0 {
+ continue
+ }
+ other := head.OrderByItems[0].Item
+ if detectJSONType(other) != class {
+ continue
+ }
+ order, err := compareComposites(item, other)
+ if err != nil {
+ return err
+ }
+ if order != 0 {
+ return errors.Wrapf(ErrUnsupportedPlanFeature,
+ "gonative: ORDER BY cannot order two different %ss", class)
+ }
+ }
+ return nil
+}
+
+// windowFull reports whether LIMIT, or the TOP folded into it, is satisfied.
+func (p *orderByPipeline) windowFull() bool {
+ return p.limit >= 0 && p.emitted >= p.limit
+}
+
// firstSighting reports whether a DISTINCT lets this payload through, and
// records it. Always true for a plan without a DISTINCT.
func (p *orderByPipeline) firstSighting(payload json.RawMessage) (bool, error) {
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
index a886d62c12..8c920fbf24 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go
@@ -586,3 +586,166 @@ func TestOrderByPipeline_DistinctDropsDuplicatesBeforeTheLimit(t *testing.T) {
})
assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`}, rows)
}
+
+// A merge is only correct when its comparator agrees with the order inside the
+// streams it merges, and inside the array and object classes nothing derived
+// from the values does: the service answers these three [1,2], [2], [1,3],
+// while canonical JSON would put them [1,2], [1,3], [2] — an order that is
+// neither, and that a TOP 2 would cut in the wrong place.
+func TestOrderByPipeline_RefusesAnOrderItCannotMerge(t *testing.T) {
+ p := newAscPipeline(t, "0", "1")
+
+ res, err := p.Run()
+ require.NoError(t, err)
+ require.NotEmpty(t, res.Requests)
+ feed := map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `[1,2]`, payload: `{"x":[1,2]}`},
+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),
+ "1": mkOrderByBody(t, ordered{ob: `[1,3]`, payload: `{"x":[1,3]}`}),
+ }
+ results := make([]queryengine.QueryResult, 0, len(res.Requests))
+ for _, req := range res.Requests {
+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, feed[req.PartitionKeyRangeID], ""))
+ }
+ require.NoError(t, p.ProvideData(results))
+
+ _, err = p.Run()
+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)
+}
+
+// One array among scalars is still ordered, because the class ranking is the
+// service's own: every array sorts after every string.
+func TestOrderByPipeline_OrdersOneCompositeAmongScalars(t *testing.T) {
+ p := newAscPipeline(t, "0", "1")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `"a"`, payload: `{"x":"a"}`},
+ ordered{ob: `[9]`, payload: `{"x":[9]}`}),
+ "1": mkOrderByBody(t, ordered{ob: `"b"`, payload: `{"x":"b"}`}),
+ })
+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`, `{"x":[9]}`}, rows)
+}
+
+// Rows past the window never reach the caller, so an order the engine cannot
+// produce among them must not fail the query: TOP 2 over these streams is
+// 0, 1, and the arrays behind them are nobody's business.
+func TestOrderByPipeline_StopsAtTheLimitBeforeComparingExcludedRows(t *testing.T) {
+ p := newTopOrderByPipeline(t, 2, "0", "1")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `0`, payload: `{"x":0}`},
+ ordered{ob: `[1]`, payload: `{"x":[1]}`}),
+ "1": mkOrderByBody(t,
+ ordered{ob: `1`, payload: `{"x":1}`},
+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),
+ })
+ assert.Equal(t, []string{`{"x":0}`, `{"x":1}`}, rows)
+}
+
+// TOP 0 asks for nothing, so no range is asked for a page and no row is ever
+// compared.
+func TestOrderByPipeline_AsksNothingOfRowsUnderTopZero(t *testing.T) {
+ p := newTopOrderByPipeline(t, 0, "0", "1")
+
+ res, err := p.Run()
+ require.NoError(t, err)
+ assert.Empty(t, res.Requests)
+ assert.True(t, p.IsComplete())
+}
+
+// A page in flight when the window fills is dropped rather than merged, so an
+// order the engine cannot produce inside it cannot fail a query that is already
+// answered.
+func TestOrderByPipeline_IgnoresAPageThatLandsAfterTheLimit(t *testing.T) {
+ p := newTopOrderByPipeline(t, 1, "0", "1")
+
+ res, err := p.Run()
+ require.NoError(t, err)
+ require.Len(t, res.Requests, 2)
+ require.NoError(t, p.ProvideData([]queryengine.QueryResult{
+ queryengine.NewQueryResult("0", mkOrderByBody(t, ordered{ob: `0`, payload: `{"x":0}`}), "more-0"),
+ queryengine.NewQueryResult("1", mkOrderByBody(t, ordered{ob: `1`, payload: `{"x":1}`}), "more-1"),
+ }))
+
+ flushed, err := p.Run()
+ require.NoError(t, err)
+ got := make([]string, 0, len(flushed.Items))
+ for _, item := range flushed.Items {
+ got = append(got, string(item))
+ }
+ assert.Equal(t, []string{`{"x":0}`}, got)
+ require.True(t, p.IsComplete())
+
+ require.NoError(t, p.ProvideData([]queryengine.QueryResult{
+ queryengine.NewQueryResult("0", mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1]}`}), ""),
+ queryengine.NewQueryResult("1", mkOrderByBody(t, ordered{ob: `[2]`, payload: `{"x":[2]}`}), ""),
+ }))
+
+ // Nothing more may come out of it, and nothing in that page may fail it.
+ late, err := p.Run()
+ require.NoError(t, err)
+ assert.Empty(t, late.Items)
+ assert.True(t, p.IsComplete())
+}
+
+// Three ranges headed by a scalar and two arrays: the first row is the scalar
+// whatever the arrays' order, so TOP 1 is answerable.
+func TestOrderByPipeline_AnswersWhenTheFirstRowIsUnambiguous(t *testing.T) {
+ p := newTopOrderByPipeline(t, 1, "0", "1", "2")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t, ordered{ob: `0`, payload: `{"x":0}`}),
+ "1": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1]}`}),
+ "2": mkOrderByBody(t, ordered{ob: `[2]`, payload: `{"x":[2]}`}),
+ })
+ assert.Equal(t, []string{`{"x":0}`}, rows)
+}
+
+// One range's own stream is already in the service's order, so two different
+// arrays inside it are no trouble at all.
+func TestOrderByPipeline_OrdersCompositesWithinOneRange(t *testing.T) {
+ p := newAscPipeline(t, "0")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t,
+ ordered{ob: `[1]`, payload: `{"x":[1]}`},
+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),
+ })
+ assert.Equal(t, []string{`{"x":[1]}`, `{"x":[2]}`}, rows)
+}
+
+// Two ranges holding the same value tie, and a tie is not an ambiguity.
+func TestOrderByPipeline_OrdersEqualCompositesAcrossRanges(t *testing.T) {
+ p := newAscPipeline(t, "0", "1")
+ rows := runOnePass(t, p, map[string][]byte{
+ "0": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1],"r":0}`}),
+ "1": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1],"r":1}`}),
+ })
+ assert.Len(t, rows, 2)
+}
+
+// The ambiguity can surface mid-stream, once the scalars ahead of it are gone.
+func TestOrderByPipeline_RefusesAnAmbiguityBehindTheScalars(t *testing.T) {
+ p := newAscPipeline(t, "0", "1")
+
+ res, err := p.Run()
+ require.NoError(t, err)
+ results := []queryengine.QueryResult{
+ queryengine.NewQueryResult("0", mkOrderByBody(t,
+ ordered{ob: `0`, payload: `{"x":0}`},
+ ordered{ob: `[1,2]`, payload: `{"x":[1,2]}`}), ""),
+ queryengine.NewQueryResult("1", mkOrderByBody(t,
+ ordered{ob: `1`, payload: `{"x":1}`},
+ ordered{ob: `[1,3]`, payload: `{"x":[1,3]}`}), ""),
+ }
+ require.Len(t, res.Requests, 2)
+ require.NoError(t, p.ProvideData(results))
+
+ var failure error
+ for !p.IsComplete() {
+ if _, err := p.Run(); err != nil {
+ failure = err
+ break
+ }
+ }
+ assert.ErrorIs(t, failure, ErrUnsupportedPlanFeature)
+}
diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go b/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go
index ff2cb2d33b..524eee6a40 100644
--- a/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go
+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go
@@ -2,8 +2,18 @@ package gonative
import "errors"
-// ErrUnsupportedPlanFeature reports that the gateway's query plan needs a
-// feature this engine does not implement, so the caller should surface the
-// gateway's own error rather than the engine's. Upstream's queryengine package
-// defines no such sentinel, so it lives with the engine that returns it.
+// ErrUnsupportedPlanFeature reports that the engine cannot answer the query as
+// asked. It comes back two ways, and the difference matters to the caller.
+//
+// From CreateQueryPipeline, before any row is read: the plan needs a feature
+// the engine does not implement, and the driver can still surface the
+// gateway's own error for the query.
+//
+// From a running pipeline, once the rows show what the plan did not: an ORDER
+// BY over two values the service orders by its index's own encoding. Nothing
+// is left to fall back to, so this one reaches the user as the engine's error
+// — which is the point, since the alternative is a wrong order.
+//
+// Upstream's queryengine package defines no such sentinel, so it lives with
+// the engine that returns it.
var ErrUnsupportedPlanFeature = errors.New("cosmosdb/queryengine: unsupported plan feature")