Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(cosmosdb): group and order cross-partition queries the way the service does (#21897)
Danny Xu5f5257e2c8d1
diff --git a/backend/plugin/db/cosmosdb/cosmosdb_test.go b/backend/plugin/db/cosmosdb/cosmosdb_test.goindex 2a14ad5f3a..9f4f6edd24 100644--- a/backend/plugin/db/cosmosdb/cosmosdb_test.go+++ b/backend/plugin/db/cosmosdb/cosmosdb_test.go@@ -642,6 +642,43 @@ func pagedRows() int {return len(pagedBigBuckets)*pagedBigRows + len(pagedSmallBuckets)*pagedSmallRows}+// pagedShapes gives eleven of the fixture's rows a `v`, one per shape a+// document can hold, chosen so that the values span the container's ranges:+// two rows in each big bucket and one in each small one. Every other row+// leaves `v` undefined, so a query grouping or ordering on it meets both.+//+// Ordering them is what the service does by type — undefined < null < bool <+// number < string < array < object — and the two seqs inside each of the array+// and object classes are interchangeable, since the service orders those by+// its index's own encoding.+var pagedShapes = []struct {+ seq int+ value string+}{+ {seq: 0, value: `null`},+ {seq: 1, value: `false`},+ {seq: pagedBigRows, value: `true`},+ {seq: pagedBigRows + 1, value: `1`},+ {seq: 2 * pagedBigRows, value: `20`},+ {seq: 2*pagedBigRows + 1, value: `"a"`},+ {seq: 2*pagedBigRows + pagedSmallRows, value: `"b"`},+ {seq: 2*pagedBigRows + 2*pagedSmallRows, value: `[1,2]`},+ {seq: 2*pagedBigRows + 3*pagedSmallRows, value: `[2]`},+ {seq: 2*pagedBigRows + 4*pagedSmallRows, value: `{"a":1}`},+ {seq: 2*pagedBigRows + 5*pagedSmallRows, value: `{"b":1}`},+}++// pagedShapeOf returns the shape attached to a row, or "" when the row leaves+// `v` undefined.+func pagedShapeOf(seq int) string {+ for _, shape := range pagedShapes {+ if shape.seq == seq {+ return shape.value+ }+ }+ return ""+}+var (pagedOnce sync.OncepagedContainerClient *azcosmos.ContainerClient@@ -712,11 +749,15 @@ func seedPaged(driver *Driver) (*azcosmos.ContainerClient, int, error) {for range 12 {wg.Go(func() {for r := range rows {- item, err := json.Marshal(pagedRow{+ row := pagedRow{ID: fmt.Sprintf("row-%d", r.seq),Bucket: r.bucket,Seq: r.seq,- })+ }+ if shape := pagedShapeOf(r.seq); shape != "" {+ row.V = json.RawMessage(shape)+ }+ item, err := json.Marshal(row)if err == nil {err = seedItem(ctx, container, r.bucket, item)}@@ -748,6 +789,10 @@ type pagedRow struct {ID string `json:"id"`Bucket string `json:"bucket"`Seq int `json:"seq"`+ // V carries one of pagedShapes on the rows that have one, and is absent+ // from every other document — which is what makes the fixture answer for+ // a column a schemaless store defines on only some of its rows.+ V json.RawMessage `json:"v,omitempty"`}// pagedSeqs runs a query against the multi-page fixture and returns the seq of@@ -1275,3 +1320,56 @@ func TestCrossPartitionDistinctWithOrderByThroughTheDriver(t *testing.T) {}assert.Equal(t, slices.Sorted(slices.Values(crossPartitionCountries)), got)}++// TestCrossPartitionSparseGroupBy groups on a column the documents do not all+// define, which is ordinary in a schemaless store. The service answers the+// undefined rows as a group of their own, counted apart from an explicit null+// and spelled with the alias left out of the row.+func TestCrossPartitionSparseGroupBy(t *testing.T) {+ container := pagedFixture(t)++ want := []string{fmt.Sprintf(`{"n":%d}`, pagedRows()-len(pagedShapes))}+ for _, shape := range pagedShapes {+ want = append(want, fmt.Sprintf(`{"k":%s,"n":1}`, shape.value))+ }+ rows := pagedQuery(t, container, `SELECT c.v AS k, COUNT(1) AS n FROM c GROUP BY c.v`)+ assert.ElementsMatch(t, want, rows)++ // The groups also come back in key order, the undefined one first, which+ // is what makes two runs of one query agree.+ require.NotEmpty(t, rows)+ assert.Equal(t, want[0], rows[0])+ assert.ElementsMatch(t, []string{`{"k":{"a":1},"n":1}`, `{"k":{"b":1},"n":1}`}, rows[len(rows)-2:],+ "objects sort last")+}++// TestCrossPartitionMixedTypeOrder checks ORDER BY against the order the+// service itself returns: undefined < null < bool < number < string < array <+// object.+//+// Two different arrays, or two different objects, it cannot order when they+// reach the merge from different ranges, because the service orders those by+// its index's own encoding — [1,2] < [2] < [1,3] on a real container. Which+// range holds which value is the emulator's to decide, so the engine's refusal+// is covered in merge_test.go, where the ranges are given.+func TestCrossPartitionMixedTypeOrder(t *testing.T) {+ container := pagedFixture(t)++ scalars := make([]int, 0, len(pagedShapes))+ for _, shape := range pagedShapes {+ if shape.value[0] != '[' && shape.value[0] != '{' {+ scalars = append(scalars, shape.seq)+ }+ }+ got := pagedSeqs(t, container,+ `SELECT c.seq FROM c WHERE IS_DEFINED(c.v) AND NOT IS_ARRAY(c.v) AND NOT IS_OBJECT(c.v) ORDER BY c.v`)+ assert.Equal(t, scalars, got, "the scalar classes are ordered as the service orders them")++ // One composite among them still orders, since the class ranking is the+ // service's own: an array sorts after every string.+ withOneArray := append(slices.Clone(scalars), pagedShapes[7].seq)+ got = pagedSeqs(t, container, fmt.Sprintf(+ `SELECT c.seq FROM c WHERE IS_DEFINED(c.v) AND NOT IS_OBJECT(c.v) AND (NOT IS_ARRAY(c.v) OR c.seq = %d) ORDER BY c.v`,+ pagedShapes[7].seq))+ assert.Equal(t, withOneArray, got, "an array sorts after the scalars")+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.goindex a08a7ddfdb..2f8ca69659 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/agg_test.go@@ -202,3 +202,21 @@ func TestNumberRawMessage_IntegerFormatting(t *testing.T) {assert.Equal(t, json.RawMessage(`0`), numberRawMessage(0))assert.Equal(t, json.RawMessage(`1.5`), numberRawMessage(1.5))}++// MIN and MAX fold their partials through compareItems, so the composite+// classes reach them too: two ranges reporting the same array agree on the+// answer, while two reporting different arrays have none the values can give.+func TestMinAgg_FoldsEqualComposites(t *testing.T) {+ out := feedFinal(t, newAggregator("Min"),+ []byte(`{"min":[1,2],"count":2}`),+ []byte(`{"min":[1,2],"count":3}`),+ )+ assert.Equal(t, json.RawMessage(`[1,2]`), out)+}++func TestMinAgg_RefusesTwoDifferentComposites(t *testing.T) {+ a := newAggregator("Min")+ require.NoError(t, a.feedPartial([]byte(`{"min":[1,2],"count":1}`)))+ err := a.feedPartial([]byte(`{"min":[1,3],"count":1}`))+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.goindex 1cf3cadcdb..63995a1a70 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp.go@@ -10,10 +10,32 @@ import (// jsonType enumerates the Cosmos item-ordering classes in ascending order://-// undefined < null < bool < number < string+// undefined < null < bool < number < string < array < object//-// Arrays and objects are not orderable in the Cosmos item-comparison rules;-// callers that encounter them should treat the result as an error.+// That is the order the service itself returns mixed types in, and the class+// ranking is reproducible. The order *within* the array and object classes is+// not: it is the index's own encoding, and a real container answers+// [1,2] < [2] < [1,3] and {"b":1} < {"a":2} < {"a":1}, which no function of the+// values gives back.+//+// So two unequal values of one of those classes have no order here, and the+// two comparators below part ways over them. compareItems fails, for the+// callers whose answer is the order itself: an ORDER BY that answered from a+// comparator disagreeing with the per-partition streams it merges would put+// out rows in an order that is neither the service's nor its own, and under a+// TOP it would take the wrong rows outright; MIN and MAX fold their partials+// through the same comparison and would pick an arbitrary one of two arrays.+// compareItemsTotal instead orders them by canonical JSON, which is+// repeatable but not the service's, so a caller may use it only where that+// cannot reach the answer:+//+// - a GROUP BY the user did not order, whose rows have no order to be wrong+// about;+// - the ORDER BY merge heap, which needs *a* minimum to pop at all. Its+// order is provisional, and ambiguousFirst then asks the question that+// matters of the row about to go out — so changing the heap to+// compareItems would not make it stricter, only fail rows the answer+// never depended on.type jsonType intconst (@@ -22,7 +44,8 @@ const (jsonBooljsonNumberjsonString- jsonUnorderable // arrays, objects — reject rather than compare+ jsonArray+ jsonObject)// detectJSONType inspects raw JSON (possibly leading whitespace) and returns@@ -39,8 +62,10 @@ func detectJSONType(raw json.RawMessage) jsonType {return jsonBoolcase '"':return jsonString- case '[', '{':- return jsonUnorderable+ case '[':+ return jsonArray+ case '{':+ return jsonObject}// digit, minus, plus, or dot → numberif (trimmed[0] >= '0' && trimmed[0] <= '9') || trimmed[0] == '-' || trimmed[0] == '+' || trimmed[0] == '.' {@@ -49,14 +74,16 @@ func detectJSONType(raw json.RawMessage) jsonType {return jsonUndefined}-// compareItems compares two JSON-encoded scalars using Cosmos item ordering.+// compareItems compares two JSON-encoded values using Cosmos item ordering.// Return values follow the usual convention: -1, 0, +1.-// Returns an error when either side is unorderable (array/object).+//+// Values of different classes always compare, and so do two equal values of+// any class. Two *unequal* arrays, or two unequal objects, do not: the service+// orders those by its index's own encoding, which no function of the values+// reproduces, and the error says so by wrapping ErrUnsupportedPlanFeature.+// Callers that may order such a pair take compareItemsTotal instead.func compareItems(a, b json.RawMessage) (int, error) {ta, tb := detectJSONType(a), detectJSONType(b)- if ta == jsonUnorderable || tb == jsonUnorderable {- return 0, errors.New("gonative: cannot compare unorderable values (arrays/objects)")- }if ta != tb {if ta < tb {return -1, nil@@ -109,7 +136,69 @@ func compareItems(a, b json.RawMessage) (int, error) {return 0, errors.Wrap(err, "gonative: cmp string b")}return strings.Compare(av, bv), nil+ case jsonArray, jsonObject:+ order, err := compareComposites(a, b)+ if err != nil {+ return 0, err+ }+ if order == 0 {+ return 0, nil+ }+ // Equal composites are one value and may be ordered together; unequal+ // ones the service separates by an encoding this cannot see.+ return 0, errors.Wrapf(ErrUnsupportedPlanFeature,+ "gonative: cannot order two different %ss", ta)default:return 0, errors.Errorf("gonative: unknown jsonType %d", ta)}}++// compareItemsTotal is compareItems extended to a total order: two unequal+// values of one composite class are ordered by their canonical JSON. That+// order is not the service's, so a caller either has to be free to pick one —+// a GROUP BY nobody ordered, or a sort key appended to settle ties — or has to+// check the rows it hands back, as the merge heap does through+// ambiguousFirst. See jsonType.+func compareItemsTotal(a, b json.RawMessage) (int, error) {+ ta, tb := detectJSONType(a), detectJSONType(b)+ if ta == tb && (ta == jsonArray || ta == jsonObject) {+ return compareComposites(a, b)+ }+ return compareItems(a, b)+}++// compareComposites orders two arrays or two objects by their canonical JSON,+// which is 0 exactly when they are the same value.+func compareComposites(a, b json.RawMessage) (int, error) {+ ca, err := canonicalJSON(a)+ if err != nil {+ return 0, errors.Wrap(err, "gonative: cmp composite a")+ }+ cb, err := canonicalJSON(b)+ if err != nil {+ return 0, errors.Wrap(err, "gonative: cmp composite b")+ }+ return bytes.Compare(ca, cb), nil+}++// String names the class in an error a user may read.+func (t jsonType) String() string {+ switch t {+ case jsonUndefined:+ return "undefined"+ case jsonNull:+ return "null"+ case jsonBool:+ return "boolean"+ case jsonNumber:+ return "number"+ case jsonString:+ return "string"+ case jsonArray:+ return "array"+ case jsonObject:+ return "object"+ default:+ return "value"+ }+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.goindex 36248c3fc2..6f9308b8da 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/cmp_test.go@@ -2,6 +2,7 @@ package gonativeimport ("encoding/json"+ "slices""testing""github.com/stretchr/testify/assert"@@ -55,11 +56,85 @@ func TestCompareItems_WithinType(t *testing.T) {}}-func TestCompareItems_UnorderableFails(t *testing.T) {- _, err := compareItems(json.RawMessage(`[1,2]`), json.RawMessage(`3`))- require.Error(t, err)- _, err = compareItems(json.RawMessage(`{}`), json.RawMessage(`null`))- require.Error(t, err)+// The service sorts mixed types undefined < null < bool < number < string <+// array < object; measured on a container holding one row of each.+func TestCompareItems_RanksCompositesLast(t *testing.T) {+ for _, tc := range []struct {+ name string+ a, b json.RawMessage+ wantCmp int+ }{+ {"array after number", json.RawMessage(`[1,2]`), json.RawMessage(`3`), 1},+ {"array after string", json.RawMessage(`[1,2]`), json.RawMessage(`"zzz"`), 1},+ {"object after array", json.RawMessage(`{"a":1}`), json.RawMessage(`[9,9]`), 1},+ {"object after null", json.RawMessage(`{}`), json.RawMessage(`null`), 1},+ {"array before object", json.RawMessage(`[]`), json.RawMessage(`{}`), -1},+ {"equal arrays", json.RawMessage(`[1,2]`), json.RawMessage(`[1,2]`), 0},+ {"equal objects, keys reordered", json.RawMessage(`{"a":1,"b":2}`), json.RawMessage(`{"b":2,"a":1}`), 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, err := compareItems(tc.a, tc.b)+ require.NoError(t, err)+ assert.Equal(t, tc.wantCmp, got)+ back, err := compareItems(tc.b, tc.a)+ require.NoError(t, err)+ assert.Equal(t, -tc.wantCmp, back, "comparison must be antisymmetric")+ })+ }+}++// Two different arrays, or two different objects, have no order the values can+// give back: an ORDER BY merge that picked one would disagree with the streams+// it merges, and under a TOP it would return the wrong rows.+func TestCompareItems_RefusesTwoDifferentComposites(t *testing.T) {+ for _, tc := range []struct {+ name string+ a, b json.RawMessage+ }{+ {"two arrays", json.RawMessage(`[1,2]`), json.RawMessage(`[1,3]`)},+ {"two objects", json.RawMessage(`{"a":1}`), json.RawMessage(`{"b":1}`)},+ {"nested", json.RawMessage(`[[1]]`), json.RawMessage(`[[2]]`)},+ } {+ t.Run(tc.name, func(t *testing.T) {+ _, err := compareItems(tc.a, tc.b)+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+ _, err = compareItems(tc.b, tc.a)+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+ })+ }+}++// compareItemsTotal is the one for a caller free to pick an order: it must be+// total and the same on every run.+func TestCompareItemsTotal_OrdersCompositesConsistently(t *testing.T) {+ values := []json.RawMessage{+ json.RawMessage(`[1,3]`), json.RawMessage(`[2]`), json.RawMessage(`[1,2]`),+ json.RawMessage(`{"b":1}`), json.RawMessage(`{"a":2}`), json.RawMessage(`{"a":1}`),+ }+ sortOnce := func() []string {+ shuffled := slices.Clone(values)+ slices.SortStableFunc(shuffled, func(a, b json.RawMessage) int {+ order, err := compareItemsTotal(a, b)+ require.NoError(t, err)+ return order+ })+ out := make([]string, 0, len(shuffled))+ for _, v := range shuffled {+ out = append(out, string(v))+ }+ return out+ }+ first := sortOnce()+ assert.Equal(t, first, sortOnce())+ // Whatever the order within each class, every array precedes every object.+ for i, v := range first {+ if v[0] == '{' {+ for _, rest := range first[i:] {+ assert.Equal(t, byte('{'), rest[0], "an array sorted after an object: %v", first)+ }+ break+ }+ }}func TestCompareItems_HandlesLeadingWhitespace(t *testing.T) {@@ -81,8 +156,8 @@ func TestDetectJSONType(t *testing.T) {{`42`, jsonNumber},{`-1.5`, jsonNumber},{`"x"`, jsonString},- {`[1,2]`, jsonUnorderable},- {`{"a":1}`, jsonUnorderable},+ {`[1,2]`, jsonArray},+ {`{"a":1}`, jsonObject},{` "pad" `, jsonString},}for _, tc := range cases {diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go b/backend/plugin/db/cosmosdb/queryengine/gonative/doc.goindex 5cef1119ab..266605bf17 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/doc.go@@ -4,6 +4,7 @@// The CosmosDB driver engages it per query: the gateway runs what it can on// its own, and only a cross-partition query it refuses is replayed through// this engine. SupportedFeatures lists the operators the engine implements;-// a plan that needs anything else fails with ErrUnsupportedPlanFeature, and-// the caller then surfaces the gateway's original error.+// a plan that needs anything else fails with ErrUnsupportedPlanFeature before+// a row is read, and the caller then surfaces the gateway's original error. A+// pipeline already running can fail with it too — see that sentinel.package gonativediff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group.goindex e13f1ea13b..43c3dbecdd 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group.go@@ -1,8 +1,10 @@package gonativeimport (+ "bytes""encoding/json""slices"+ "strings""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine""github.com/pkg/errors"@@ -78,16 +80,9 @@ type groupPipeline struct {// is what the output falls back to when the keys cannot be ordered.order []string- // descending says which way to order the output rows, and- // directionKnown whether the data has told us yet. A GROUP BY the user- // also ordered- // reaches the engine as a plan with no order fields at all — the ORDER BY- // survives only inside rewrittenQuery — so the direction can be read only- // off the responses: the gateway returns each partition's own groups in- // the requested order, and the first adjacent pair that differs says- // which way they run.- descending bool- directionKnown bool+ // sort is how to order the output rows: which group key to compare+ // first, which way, and how strictly — see groupSortKeys.+ sort []groupSortKey// pending holds fully finalized rows awaiting the caller's next Run().pending [][]byte@@ -130,6 +125,11 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, error}}+ sortKeys, err := groupSortKeys(qi)+ if err != nil {+ return nil, err+ }+return &groupPipeline{// GROUP BY plans always carry a non-empty rewrittenQuery that wraps the// user's projection into {groupByItems, payload}; running the original@@ -139,6 +139,7 @@ func newGroupPipeline(plan *planDoc, pkRangeIDs []string) (*groupPipeline, erroraliases: aliases,kinds: kinds,groups: make(map[string]*groupState),+ sort: sortKeys,}, nil}@@ -193,7 +194,6 @@ func (p *groupPipeline) ProvideData(results []queryengine.QueryResult) error {if err != nil {return err}- p.noteOrdering(resp.Documents)for docIdx, doc := range resp.Documents {if len(doc.GroupByItems) == 0 {return errors.Errorf("gonative: group doc %d missing groupByItems", docIdx)@@ -260,7 +260,9 @@ func (p *groupPipeline) initGroupState(doc groupDoc) *groupState {// sortGroups. Walking p.groups instead would emit them in Go's randomized map// order, which made the same query answer in a different order every run.func (p *groupPipeline) finalize() error {- p.sortGroups()+ if err := p.sortGroups(); err != nil {+ return err+ }for _, key := range p.order {row, err := p.buildRow(p.groups[key])if err != nil {@@ -272,70 +274,231 @@ func (p *groupPipeline) finalize() error {return nil}-// noteOrdering reads the output direction off one partition's response, which-// is the only place a GROUP BY's ORDER BY reaches the engine. Gives up on a-// key it cannot compare: sortGroups checks for those itself.-func (p *groupPipeline) noteOrdering(docs []groupDoc) {- if p.directionKnown {- return+// orderByKeyword is how the gateway spells the clause in a rewritten query.+const orderByKeyword = "ORDER BY"++// orderByKey is one key of an ORDER BY clause.+type orderByKey struct {+ expr string+ descending bool+}++// groupSortKey names one group key to order the output rows by: its position+// in groupByItems, which way to compare it, and whether the query named it.+//+// A key the query named has to be compared the service's way or not at all. A+// key appended to settle ties the clause says nothing about takes any+// repeatable order, so it is compared with compareItemsTotal and never fails.+type groupSortKey struct {+ index int+ descending bool+ named bool+}++// groupSortKeys works out how to order a GROUP BY's output rows. A GROUP BY+// the user also ordered reaches the engine as a plan with no order fields at+// all — orderBy and orderByExpressions stay empty and the ORDER BY survives+// only inside the rewritten query — so the clause is read from that text.+//+// The engine orders these rows by comparing their group keys, so every key the+// clause names must be one of the GROUP BY expressions. An order it cannot+// produce that way — by an aggregate, or by a column the query does not group+// by — fails the query rather than answering in an order nobody asked for. The+// gateway refuses those queries itself today, so this is the engine holding the+// line rather than a reachable error.+//+// Group keys the clause leaves out are appended ascending. The clause does not+// constrain them, and ordering them too is what makes two runs of one query+// answer alike.+func groupSortKeys(qi planQueryInfo) ([]groupSortKey, error) {+ keys, found := trailingOrderBy(qi.RewrittenQuery)+ if found && len(keys) == 0 {+ return nil, errors.Wrap(ErrUnsupportedPlanFeature, "ORDER BY is not a plain list of keys")}- for i := 1; i < len(docs); i++ {- order, err := compareGroupKeys(docs[i-1].GroupByItems, docs[i].GroupByItems)- if err != nil {- return+ sortKeys := make([]groupSortKey, 0, len(qi.GroupByExpressions))+ named := make([]bool, len(qi.GroupByExpressions))+ for _, key := range keys {+ index := slices.Index(qi.GroupByExpressions, key.expr)+ if index < 0 {+ return nil, errors.Wrapf(ErrUnsupportedPlanFeature,+ "ORDER BY key %q is not one of the GROUP BY expressions", key.expr)}- if order != 0 {- p.descending = order > 0- p.directionKnown = true- return+ if named[index] {+ continue+ }+ named[index] = true+ sortKeys = append(sortKeys, groupSortKey{index: index, descending: key.descending, named: true})+ }+ for index, taken := range named {+ if !taken {+ sortKeys = append(sortKeys, groupSortKey{index: index})}}+ return sortKeys, nil}-// sortGroups puts p.order in group-key order, the direction taken from the-// gateway's own per-partition streams. A GROUP BY without an ORDER BY has no-// defined output order, so ordering those rows too costs nothing and is what-// makes every run answer the same way.+// trailingOrderBy splits the ORDER BY clause that ends query into its keys.+// found is false when the query has no trailing ORDER BY; keys is nil with+// found true when the clause is not the plain `<expression> [ASC|DESC]` list+// the gateway writes there, which the caller must treat as an order it cannot+// honour rather than as no order at all.//-// Leaves first-seen order in place when any group key is a shape Cosmos-// cannot order (an array or an object), since the comparator would have to-// invent an answer for those.-func (p *groupPipeline) sortGroups() {- for _, key := range p.order {- for _, item := range p.groups[key].items {- if detectJSONType(item.Item) == jsonUnorderable {- return+// The words are matched outside quoted text, so a property path that needs+// bracket notation — c["order by"] — is a key and not a clause, whichever it+// is spelled as.+//+// The keys are plain property paths because that is all the service allows a+// grouped query to order by: `GROUP BY SUBSTRING(c.x, 0, 1) ORDER BY+// SUBSTRING(c.x, 0, 1)` is refused outright with SC2103, "not contained in+// either an aggregate function or the GROUP BY clause", however exactly the+// two spellings match. So a key here holds no comma and no bracketed+// arithmetic of its own.+func trailingOrderBy(query string) (keys []orderByKey, found bool) {+ at := lastUnquotedIndex(query, orderByKeyword)+ if at < 0 {+ return nil, false+ }+ tail := query[at+len(orderByKeyword):]+ for _, item := range splitUnquoted(tail, ',') {+ fields := splitUnquoted(item, ' ', '\t', '\n', '\r')+ var key orderByKey+ switch len(fields) {+ case 1:+ key.expr = fields[0]+ case 2:+ key.expr = fields[0]+ switch strings.ToUpper(fields[1]) {+ case "ASC":+ case "DESC":+ key.descending = true+ default:+ return nil, true}+ default:+ return nil, true}+ keys = append(keys, key)}- slices.SortStableFunc(p.order, func(a, b string) int {- order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items)- if err != nil {- // Unreachable: every key was checked orderable just above.- return 0+ if len(keys) == 0 {+ return nil, true+ }+ return keys, true+}++// lastUnquotedIndex returns the last index at which needle occurs in s outside+// quoted text, ignoring case, or -1.+func lastUnquotedIndex(s, needle string) int {+ quoted := quotedBytes(s)+ for i := len(s) - len(needle); i >= 0; i-- {+ if !quoted[i] && strings.EqualFold(s[i:i+len(needle)], needle) {+ return i+ }+ }+ return -1+}++// splitUnquoted splits s on any of seps that sits outside quoted text,+// dropping empty pieces so runs of whitespace behave like one separator.+func splitUnquoted(s string, seps ...byte) []string {+ quoted := quotedBytes(s)+ var out []string+ start := -1+ for i := range len(s) {+ if !quoted[i] && slices.Contains(seps, s[i]) {+ if start >= 0 {+ out = append(out, s[start:i])+ start = -1+ }+ continue+ }+ if start < 0 {+ start = i+ }+ }+ if start >= 0 {+ out = append(out, s[start:])+ }+ return out+}++// quotedBytes marks the bytes of s that sit inside quoted text. A rewritten+// query quotes a property path spelled in bracket notation and any string+// literal the query compares against, and the words of a clause mean nothing+// inside either.+func quotedBytes(s string) []bool {+ quoted := make([]bool, len(s))+ var quote byte+ for i := 0; i < len(s); i++ {+ c := s[i]+ if quote == 0 {+ if c == '"' || c == '\'' {+ quote = c+ quoted[i] = true+ }+ continue+ }+ quoted[i] = true+ if c == '\\' && i+1 < len(s) {+ // An escaped quote does not close the span, and the byte it+ // escapes is inside it either way.+ i+++ quoted[i] = true+ continue+ }+ if c == quote {+ quote = 0}- if p.descending {- return -order+ }+ return quoted+}++// sortGroups puts p.order in group-key order, in the direction the query asked+// for. Each key carries how strictly it is compared — see groupSortKey — so a+// key the query named fails rather than take an invented order, while one+// appended to settle ties takes any repeatable one.+func (p *groupPipeline) sortGroups() error {+ var failed error+ slices.SortStableFunc(p.order, func(a, b string) int {+ order, err := compareGroupKeys(p.groups[a].items, p.groups[b].items, p.sort)+ if err != nil && failed == nil {+ failed = err}return order})+ return failed}-// compareGroupKeys orders two group keys under Cosmos item ordering, column-// by column.-func compareGroupKeys(a, b []groupByItem) (int, error) {- for i := range min(len(a), len(b)) {- order, err := compareItems(a[i].Item, b[i].Item)+// compareGroupKeys orders two group keys under Cosmos item ordering, comparing+// the columns sortKeys names in the order, direction and strictness it gives.+func compareGroupKeys(a, b []groupByItem, sortKeys []groupSortKey) (int, error) {+ for _, key := range sortKeys {+ if key.index >= len(a) || key.index >= len(b) {+ continue+ }+ cmp := compareItemsTotal+ if key.named {+ cmp = compareItems+ }+ order, err := cmp(a[key.index].Item, b[key.index].Item)if err != nil {return 0, err}if order != 0 {+ if key.descending {+ return -order, nil+ }return order, nil}}return len(a) - len(b), nil}+// isUndefinedItem reports whether a groupByItems entry carries no value,+// which is how the gateway spells a key the document does not define.+func isUndefinedItem(raw json.RawMessage) bool {+ return len(bytes.TrimLeft(raw, " \t\n\r")) == 0+}+// buildRow assembles the final JSON object for a single group, preserving// the alias order from groupByAliases. JSON object key order in Go's encoder// depends on sort for map marshaling, so we write the object manually.@@ -347,11 +510,15 @@ func (p *groupPipeline) buildRow(s *groupState) ([]byte, error) {var buf []bytebuf = append(buf, '{')for i, alias := range p.aliases {- value := json.RawMessage("null")+ var value json.RawMessageif p.kinds[i] == "" {- if v, ok := s.keyValues[alias]; ok {- value = v+ v, ok := s.keyValues[alias]+ if !ok {+ // The group's key is undefined here, and the service answers+ // that with the alias left out of the row.+ continue}+ value = v} else {final, err := s.aggregators[i].finalize()if err != nil {@@ -383,13 +550,22 @@ func groupKey(items []groupByItem) (string, error) {// Build a slice of the inner "item" values, canonicalize each, then join.// Using a combined key of canonicalized element JSON keeps equality// deterministic even for complex multi-key GROUP BY.+ //+ // A key the documents leave undefined arrives with no "item" at all. It is+ // a group of its own — the service counts it apart from an explicit null —+ // so each part is wrapped in an array and an undefined one is the empty+ // array, which no value's canonical form can collide with.parts := make([]json.RawMessage, len(items))for i, it := range items {+ if isUndefinedItem(it.Item) {+ parts[i] = json.RawMessage(`[]`)+ continue+ }c, err := canonicalJSON(it.Item)if err != nil {return "", errors.Wrapf(err, "gonative: canonicalize group key part %d", i)}- parts[i] = c+ parts[i] = append(append(json.RawMessage{'['}, c...), ']')}b, err := json.Marshal(parts)if err != nil {diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.goindex bd4ffd12c9..40e952741e 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/group_test.go@@ -2,6 +2,8 @@ package gonativeimport ("encoding/json"+ "fmt"+ "strings""testing""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"@@ -41,6 +43,14 @@ func mkGroupBody(t *testing.T, entries ...struct {return b}+// mkGroupBodyRaw builds a partition response from literal document JSON, for+// shapes mkGroupBody cannot spell — a groupByItems entry with no "item" at+// all, which is how the gateway reports a key the document leaves undefined.+func mkGroupBodyRaw(t *testing.T, docs ...string) []byte {+ t.Helper()+ return []byte(`{"Documents":[` + strings.Join(docs, ",") + `]}`)+}+type groupEntry = struct {key, payload string}@@ -354,41 +364,259 @@ func countByCountryPlan() *planDoc {map[string]string{"country": "", "n": "Count"})}-// A GROUP BY the user also ordered reaches the engine as a plan with no order-// fields, so the direction comes off the data: the gateway returns each-// partition's own groups in the requested order.-func TestGroupPipeline_EmitsAscendingWhenThePartitionsRunAscending(t *testing.T) {- p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})+// orderedCountByCountryPlan is countByCountryPlan for a query the user also+// ordered. Such a plan carries no order fields at all — the clause survives+// only in the rewritten query, spelled as the gateway writes it.+func orderedCountByCountryPlan(clause string) *planDoc {+ plan := countByCountryPlan()+ plan.QueryInfo.RewrittenQuery = `SELECT [{"item": c.country}] AS groupByItems, ` ++ `{"country": c.country, "n": {"item": COUNT(1)}} AS payload` + "\n" ++ `FROM c` + "\n" + `GROUP BY c.country` + "\n" + clause+ return plan+}++// The direction comes from the query, not from the order the ranges happen to+// answer in: these two feed the same rows and differ only in the clause.+func TestGroupPipeline_EmitsAscendingWhenTheQueryOrdersAscending(t *testing.T) {+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country"), []string{"0", "1"})require.NoError(t, err)- rows := driveGroup(t, p, map[string][]byte{+ rows := driveGroup(t, p, descendingCountryFeed(t))+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))+}++func TestGroupPipeline_EmitsDescendingWhenTheQueryOrdersDescending(t *testing.T) {+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country DESC"), []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, descendingCountryFeed(t))+ assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))+}++func descendingCountryFeed(t *testing.T) map[string][]byte {+ t.Helper()+ return map[string][]byte{"0": mkGroupBody(t,- groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},+ groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},),"1": mkGroupBody(t,- groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},+ groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},),+ }+}++// A range that reports a single group says nothing about which way the groups+// run, so the direction has to come from the clause.+func TestGroupPipeline_EmitsDescendingWhenEveryRangeHasOneGroup(t *testing.T) {+ p, err := newGroupPipeline(orderedCountByCountryPlan("ORDER BY c.country DESC"), []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`}),+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`}),})- assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))+ assert.Equal(t, []string{"BR", "AE"}, groupKeysOf(t, rows, "country"))+}++// An order the engine cannot produce by comparing group keys has to fail the+// query: ordering by the keys anyway would answer in an order the user did not+// ask for. The service refuses these itself today, so this is the engine+// holding the line.+func TestGroupPipeline_RejectsAnOrderItCannotProduce(t *testing.T) {+ for _, tc := range []struct {+ name string+ clause string+ }{+ {"by an aggregate", "ORDER BY COUNT(1) DESC"},+ {"by an output alias", "ORDER BY n DESC"},+ {"by a column the query does not group by", "ORDER BY c.city"},+ {"by something that is not a list of keys", "ORDER BY c.country NULLS LAST"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ _, err := newGroupPipeline(orderedCountByCountryPlan(tc.clause), []string{"0", "1"})+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+ })+ }+}++// Cosmos gives every ORDER BY key its own direction, and accepts a clause that+// names only some of the group keys.+func TestGroupPipeline_OrdersMultipleKeysEachItsOwnWay(t *testing.T) {+ feed := map[string][]byte{+ "0": mkTwoKeyGroupBody(t,+ twoKeyEntry{keys: []string{`"AE"`, `2`}, payload: `{"country":"AE","city":2,"n":{"item":7}}`},+ twoKeyEntry{keys: []string{`"AE"`, `1`}, payload: `{"country":"AE","city":1,"n":{"item":8}}`},+ ),+ "1": mkTwoKeyGroupBody(t,+ twoKeyEntry{keys: []string{`"BR"`, `1`}, payload: `{"country":"BR","city":1,"n":{"item":9}}`},+ twoKeyEntry{keys: []string{`"BR"`, `2`}, payload: `{"country":"BR","city":2,"n":{"item":6}}`},+ ),+ }++ for _, tc := range []struct {+ name string+ clause string+ want []string+ }{+ {"ascending then descending", "ORDER BY c.country, c.city DESC", []string{"AE/2", "AE/1", "BR/2", "BR/1"}},+ {"descending then ascending", "ORDER BY c.country DESC, c.city", []string{"BR/1", "BR/2", "AE/1", "AE/2"}},+ {"only the second key", "ORDER BY c.city DESC", []string{"AE/2", "BR/2", "AE/1", "BR/1"}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ p, err := newGroupPipeline(countryCityPlan(tc.clause), []string{"0", "1"})+ require.NoError(t, err)+ assert.Equal(t, tc.want, countryCityKeysOf(t, driveGroup(t, p, feed)))+ })+ }}-func TestGroupPipeline_EmitsDescendingWhenThePartitionsRunDescending(t *testing.T) {- p, err := newGroupPipeline(countByCountryPlan(), []string{"0", "1"})+// A group key the clause leaves out still gets an order, so that two runs of+// one query answer alike however the ranges happened to report their groups.+func TestGroupPipeline_OrdersKeysTheClauseLeavesOut(t *testing.T) {+ p, err := newGroupPipeline(countryCityPlan("ORDER BY c.city DESC"), []string{"0", "1"})require.NoError(t, err)rows := driveGroup(t, p, map[string][]byte{- "0": mkGroupBody(t,- groupEntry{key: `"DE"`, payload: `{"country":"DE","n":{"item":1}}`},- groupEntry{key: `"AE"`, payload: `{"country":"AE","n":{"item":2}}`},+ "0": mkTwoKeyGroupBody(t,+ twoKeyEntry{keys: []string{`"BR"`, `2`}, payload: `{"country":"BR","city":2,"n":{"item":6}}`},),- "1": mkGroupBody(t,- groupEntry{key: `"CN"`, payload: `{"country":"CN","n":{"item":3}}`},- groupEntry{key: `"BR"`, payload: `{"country":"BR","n":{"item":4}}`},+ "1": mkTwoKeyGroupBody(t,+ twoKeyEntry{keys: []string{`"AE"`, `2`}, payload: `{"country":"AE","city":2,"n":{"item":7}}`},),})- assert.Equal(t, []string{"DE", "CN", "BR", "AE"}, groupKeysOf(t, rows, "country"))+ assert.Equal(t, []string{"AE/2", "BR/2"}, countryCityKeysOf(t, rows))+}++// countryCityPlan groups by two keys, which the gateway reports in+// groupByItems in the order groupByExpressions names them.+func countryCityPlan(clause string) *planDoc {+ plan := newGroupPlan("c.country",+ []string{"country", "city", "n"},+ map[string]string{"country": "", "city": "", "n": "Count"})+ plan.QueryInfo.GroupByExpressions = []string{"c.country", "c.city"}+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.country, c.city\n" + clause+ return plan+}++// mkTwoKeyGroupBody is mkGroupBody for a two-expression GROUP BY: each+// document's groupByItems carries both keys, in groupByExpressions order.+func mkTwoKeyGroupBody(t *testing.T, entries ...struct {+ keys []string+ payload string+}) []byte {+ t.Helper()+ docs := make([]map[string]any, 0, len(entries))+ for _, e := range entries {+ var payloadObj map[string]json.RawMessage+ require.NoError(t, json.Unmarshal([]byte(e.payload), &payloadObj), "invalid payload fixture: %s", e.payload)+ items := make([]map[string]any, 0, len(e.keys))+ for _, key := range e.keys {+ items = append(items, map[string]any{"item": json.RawMessage(key)})+ }+ docs = append(docs, map[string]any{"groupByItems": items, "payload": payloadObj})+ }+ b, err := json.Marshal(map[string]any{"Documents": docs})+ require.NoError(t, err)+ return b+}++type twoKeyEntry = struct {+ keys []string+ payload string+}++func countryCityKeysOf(t *testing.T, rows [][]byte) []string {+ t.Helper()+ out := make([]string, 0, len(rows))+ for _, r := range rows {+ var obj struct {+ Country string `json:"country"`+ City int `json:"city"`+ }+ require.NoError(t, json.Unmarshal(r, &obj), "row: %s", r)+ out = append(out, fmt.Sprintf("%s/%d", obj.Country, obj.City))+ }+ return out+}++// A key whose name holds a clause keyword is still a key.+func TestGroupPipeline_ReadsAnOrderWhoseKeyNameHoldsAKeyword(t *testing.T) {+ plan := newGroupPlan("c.nowhere", []string{"nowhere", "n"}, map[string]string{"nowhere": "", "n": "Count"})+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.nowhere\nORDER BY c.nowhere DESC"++ p, err := newGroupPipeline(plan, []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"nowhere":"AE","n":{"item":2}}`}),+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"nowhere":"BR","n":{"item":4}}`}),+ })+ assert.Equal(t, []string{"BR", "AE"}, groupKeysOf(t, rows, "nowhere"))+}++// A property path that needs bracket notation is a key like any other, even+// when the property is named after the clause itself. Both plans here are what+// the service answered for `GROUP BY c["country-name"]` and+// `GROUP BY c["order by"]`.+func TestGroupPipeline_ReadsAnOrderOnABracketPath(t *testing.T) {+ for _, tc := range []struct {+ name string+ expr string+ rest string+ want []string+ }{+ {+ name: "a path that has to be quoted",+ expr: `c["country-name"]`,+ rest: "\nORDER BY c[\"country-name\"] DESC",+ want: []string{"BR", "AE"},+ },+ {+ name: "a property named after the clause",+ expr: `c["order by"]`,+ rest: "\nORDER BY c[\"order by\"] DESC",+ want: []string{"BR", "AE"},+ },+ {+ name: "a property named after the clause, unordered",+ expr: `c["order by"]`,+ rest: "",+ want: []string{"AE", "BR"},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ plan := newGroupPlan(tc.expr, []string{"k", "n"}, map[string]string{"k": "", "n": "Count"})+ plan.QueryInfo.GroupByExpressions = []string{tc.expr}+ plan.QueryInfo.RewrittenQuery = "SELECT [{\"item\": " + tc.expr + "}] AS groupByItems, " ++ "{\"k\": " + tc.expr + ", \"n\": {\"item\": COUNT(1)}} AS payload\nFROM c\nGROUP BY " + tc.expr + tc.rest++ p, err := newGroupPipeline(plan, []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBody(t, groupEntry{key: `"AE"`, payload: `{"k":"AE","n":{"item":2}}`}),+ "1": mkGroupBody(t, groupEntry{key: `"BR"`, payload: `{"k":"BR","n":{"item":4}}`}),+ })+ assert.Equal(t, tc.want, groupKeysOf(t, rows, "k"))+ })+ }+}++// The words can appear in the query body — in a string literal the user wrote+// — without the query naming an order.+func TestGroupPipeline_IgnoresOrderByInsideTheQueryBody(t *testing.T) {+ plan := countByCountryPlan()+ plan.QueryInfo.RewrittenQuery = `SELECT [{"item": c.country}] AS groupByItems, ` ++ `{"country": c.country, "n": {"item": COUNT(1)}} AS payload` + "\n" ++ `FROM c` + "\n" + `WHERE c.note != 'order by me'` + "\n" + `GROUP BY c.country`++ p, err := newGroupPipeline(plan, []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, descendingCountryFeed(t))+ assert.Equal(t, []string{"AE", "BR", "CN", "DE"}, groupKeysOf(t, rows, "country"))}// Without an ORDER BY the output order is unspecified, but it still has to be@@ -417,9 +645,10 @@ func TestGroupPipeline_OutputOrderIsStableAcrossRuns(t *testing.T) {}}-// A group key Cosmos cannot order — an array or an object — leaves the output-// in first-seen order rather than asking the comparator to invent one.-func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {+// A group key the service cannot order by value — an array or an object — is+// still ordered here, by its canonical JSON, so the same query does not answer+// in a different order on the next run.+func TestGroupPipeline_OrdersCompositeKeysConsistently(t *testing.T) {plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})p, err := newGroupPipeline(plan, []string{"0"})require.NoError(t, err)@@ -431,8 +660,109 @@ func TestGroupPipeline_UnorderableKeysKeepFirstSeenOrder(t *testing.T) {),})require.Len(t, rows, 2)- assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[0]))- assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[1]))+ assert.JSONEq(t, `{"tags":["a"],"n":2}`, string(rows[0]))+ assert.JSONEq(t, `{"tags":["b"],"n":1}`, string(rows[1]))+}++// An ORDER BY on a group key the engine cannot order — two different arrays —+// fails the query. Ordering the groups by anything else would answer in an+// order the user did not ask for, which is the whole point of reading the+// clause in the first place.+func TestGroupPipeline_RefusesAnOrderedKeyItCannotOrder(t *testing.T) {+ plan := newGroupPlan("c.tags", []string{"tags", "n"}, map[string]string{"tags": "", "n": "Count"})+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.tags\nORDER BY c.tags"+ p, err := newGroupPipeline(plan, []string{"0"})+ require.NoError(t, err)++ res, err := p.Run()+ require.NoError(t, err)+ require.NotEmpty(t, res.Requests)+ body := mkGroupBody(t,+ groupEntry{key: `["b"]`, payload: `{"tags":["b"],"n":{"item":1}}`},+ groupEntry{key: `["a"]`, payload: `{"tags":["a"],"n":{"item":2}}`},+ )+ results := make([]queryengine.QueryResult, 0, len(res.Requests))+ for _, req := range res.Requests {+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, body, ""))+ }+ require.NoError(t, p.ProvideData(results))++ _, err = p.Run()+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+}++// A clause that names one of two group keys leaves the other to settle ties.+// The query asked nothing of that key, so either tie order satisfies it and an+// array there must not fail the query — only a key the clause named has to be+// ordered the service's way.+func TestGroupPipeline_SettlesTiesOnAKeyTheClauseOmits(t *testing.T) {+ plan := newGroupPlan("c.country", []string{"country", "tags", "n"},+ map[string]string{"country": "", "tags": "", "n": "Count"})+ plan.QueryInfo.GroupByExpressions = []string{"c.country", "c.tags"}+ plan.QueryInfo.RewrittenQuery = "SELECT 1\nFROM c\nGROUP BY c.country, c.tags\nORDER BY c.country DESC"++ p, err := newGroupPipeline(plan, []string{"0"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkTwoKeyGroupBody(t,+ twoKeyEntry{keys: []string{`"AE"`, `["b"]`}, payload: `{"country":"AE","tags":["b"],"n":{"item":1}}`},+ twoKeyEntry{keys: []string{`"BR"`, `["z"]`}, payload: `{"country":"BR","tags":["z"],"n":{"item":2}}`},+ twoKeyEntry{keys: []string{`"AE"`, `["a"]`}, payload: `{"country":"AE","tags":["a"],"n":{"item":3}}`},+ ),+ })+ got := make([]string, 0, len(rows))+ for _, row := range rows {+ var object struct {+ Country string `json:"country"`+ Tags []string `json:"tags"`+ }+ require.NoError(t, json.Unmarshal(row, &object), "row: %s", row)+ got = append(got, object.Country+"/"+strings.Join(object.Tags, ","))+ }+ assert.Equal(t, []string{"BR/z", "AE/a", "AE/b"}, got)+}++// A column the documents do not all define is ordinary in a schemaless store.+// The service answers it as a group of its own, counted apart from an explicit+// null and spelled with the alias left out of the row, and sends that group's+// key as a groupByItems entry with no "item" at all — the shape this fixture+// exists for.+func TestGroupPipeline_GroupsAnUndefinedKeyOnItsOwn(t *testing.T) {+ plan := newGroupPlan("c.v", []string{"k", "n"}, map[string]string{"k": "", "n": "Count"})+ p, err := newGroupPipeline(plan, []string{"0", "1"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBodyRaw(t,+ `{"groupByItems":[{}],"payload":{"n":{"item":2}}}`,+ `{"groupByItems":[{"item":null}],"payload":{"k":null,"n":{"item":1}}}`),+ "1": mkGroupBodyRaw(t,+ `{"groupByItems":[{"item":"a"}],"payload":{"k":"a","n":{"item":3}}}`,+ `{"groupByItems":[{}],"payload":{"n":{"item":5}}}`),+ })+ require.Len(t, rows, 3)+ assert.JSONEq(t, `{"n":7}`, string(rows[0]), "the undefined group merges across ranges and omits the alias")+ assert.JSONEq(t, `{"k":null,"n":1}`, string(rows[1]), "an explicit null is its own group")+ assert.JSONEq(t, `{"k":"a","n":3}`, string(rows[2]))+}++// A multi-key GROUP BY where one key is undefined in some rows: the undefined+// slot still distinguishes the group.+func TestGroupPipeline_GroupsOnAPartlyUndefinedKey(t *testing.T) {+ plan := newGroupPlan("c.a", []string{"a", "b", "n"}, map[string]string{"a": "", "b": "", "n": "Count"})+ plan.QueryInfo.GroupByExpressions = []string{"c.a", "c.b"}+ p, err := newGroupPipeline(plan, []string{"0"})+ require.NoError(t, err)++ rows := driveGroup(t, p, map[string][]byte{+ "0": mkGroupBodyRaw(t,+ `{"groupByItems":[{"item":"x"},{}],"payload":{"a":"x","n":{"item":4}}}`,+ `{"groupByItems":[{"item":"x"},{"item":7}],"payload":{"a":"x","b":7,"n":{"item":9}}}`),+ })+ require.Len(t, rows, 2)+ assert.JSONEq(t, `{"a":"x","n":4}`, string(rows[0]))+ assert.JSONEq(t, `{"a":"x","b":7,"n":9}`, string(rows[1]))}// MIN over a group where the column is undefined in every row has no value,diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.goindex 5203d9af7e..6c81b4b397 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge.go@@ -70,8 +70,9 @@ func (s *partitionStream) pop() *orderByDoc {}// mergeHeap implements heap.Interface over partition indices. The comparator-// uses compareItems on each partition's current head's orderByItems[0], with-// the sort direction flipped for descending ORDER BY.+// uses compareItemsTotal on each partition's current head's orderByItems[0],+// with the sort direction flipped for descending ORDER BY — see Less for why+// the total comparator rather than the strict one.type mergeHeap struct {partitions []*partitionStreamidx []int // heap of partition indices@@ -85,11 +86,16 @@ func (h *mergeHeap) Less(i, j int) bool {// partitions from the heap before they can be compared.a, _ := h.partitions[h.idx[i]].peek()b, _ := h.partitions[h.idx[j]].peek()- cmp, err := compareItems(a.OrderByItems[0].Item, b.OrderByItems[0].Item)+ // The heap orders with compareItemsTotal so that it always has a minimum,+ // and because that order follows the service's class ranking the minimum's+ // class is the right one. Which of two values inside the array or object+ // class comes first the service alone knows, so merge asks that question+ // of the row it is about to emit — see ambiguousFirst — rather than of+ // every pair the heap happens to sift.+ cmp, err := compareItemsTotal(a.OrderByItems[0].Item, b.OrderByItems[0].Item)if err != nil {- // compareItems rejects unorderable shapes (arrays, objects). For- // ORDER BY scalars this should never happen; if it does, fall back to- // stable-by-partition-index order so the heap remains well-defined.+ // The bytes are not JSON; keep the heap well-defined and let the row+ // fail when it is emitted.return h.idx[i] < h.idx[j]}if h.descending {@@ -217,11 +223,11 @@ func (p *orderByPipeline) IsComplete() bool {if len(p.pending) > 0 {return false}- // LIMIT reached — the caller has everything it asked for. Any rows still- // sitting in partition streams were produced speculatively by the gateway- // (it fetched OFFSET+LIMIT rows per partition to cover any interleaving)- // and are safe to drop.- if p.limit >= 0 && p.emitted >= p.limit {+ // The caller has everything it asked for. Any rows still sitting in+ // partition streams were produced speculatively by the gateway (it fetched+ // OFFSET+LIMIT rows per partition to cover any interleaving) and are safe+ // to drop.+ if p.windowFull() {return true}if !p.fanout.allDrained() {@@ -322,6 +328,13 @@ func (p *orderByPipeline) merge() error {if p.waitingForPage() {return nil}+ // A window already satisfied ends the merge before it touches a row: a+ // page that landed after the last emission still holds rows the result+ // does not include, and merging them would hand one to the caller or ask+ // an order of them that nobody needs.+ if p.windowFull() {+ return nil+ }h := &mergeHeap{partitions: p.partitions, descending: p.descending}for i, s := range p.partitions {if _, ok := s.peek(); ok {@@ -331,10 +344,6 @@ func (p *orderByPipeline) merge() error {heap.Init(h)for h.Len() > 0 {- if p.limit >= 0 && p.emitted >= p.limit {- // LIMIT reached — the caller has everything it asked for.- return nil- }top, ok := heap.Pop(h).(int)if !ok {// Unreachable: the heap is seeded with partition indexes only.@@ -342,6 +351,9 @@ func (p *orderByPipeline) merge() error {}s := p.partitions[top]doc := s.pop()+ if err := p.ambiguousFirst(doc, top); err != nil {+ return err+ }fresh, err := p.firstSighting(doc.Payload)if err != nil {return err@@ -356,6 +368,12 @@ func (p *orderByPipeline) merge() error {p.pending = append(p.pending, append([]byte(nil), doc.Payload...))p.emitted++}+ if p.windowFull() {+ // The caller has everything it asked for. Putting the range back+ // would compare its next row against the other heads, and a row+ // outside the window has no business failing the query.+ return nil+ }if _, ok := s.peek(); ok {heap.Push(h, top)continue@@ -367,6 +385,54 @@ func (p *orderByPipeline) merge() error {return nil}+// ambiguousFirst reports an error when another range's head ties with doc for+// the position doc is about to take, in a way the engine cannot settle: the+// same composite class, a different value. The service orders within those+// classes by its index's own encoding, so which of the two comes first is not+// knowable here.+//+// Every scalar is unambiguous, and so is a composite no other head shares a+// class with — including one the heap reached only because the ranges below it+// drained. A later row of the same range is not a competitor: the service+// already placed it behind doc in that range's own stream.+func (p *orderByPipeline) ambiguousFirst(doc *orderByDoc, from int) error {+ if len(doc.OrderByItems) == 0 {+ return nil+ }+ item := doc.OrderByItems[0].Item+ class := detectJSONType(item)+ if class != jsonArray && class != jsonObject {+ return nil+ }+ for i, stream := range p.partitions {+ if i == from {+ continue+ }+ head, ok := stream.peek()+ if !ok || len(head.OrderByItems) == 0 {+ continue+ }+ other := head.OrderByItems[0].Item+ if detectJSONType(other) != class {+ continue+ }+ order, err := compareComposites(item, other)+ if err != nil {+ return err+ }+ if order != 0 {+ return errors.Wrapf(ErrUnsupportedPlanFeature,+ "gonative: ORDER BY cannot order two different %ss", class)+ }+ }+ return nil+}++// windowFull reports whether LIMIT, or the TOP folded into it, is satisfied.+func (p *orderByPipeline) windowFull() bool {+ return p.limit >= 0 && p.emitted >= p.limit+}+// firstSighting reports whether a DISTINCT lets this payload through, and// records it. Always true for a plan without a DISTINCT.func (p *orderByPipeline) firstSighting(payload json.RawMessage) (bool, error) {diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.goindex a886d62c12..8c920fbf24 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/merge_test.go@@ -586,3 +586,166 @@ func TestOrderByPipeline_DistinctDropsDuplicatesBeforeTheLimit(t *testing.T) {})assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`}, rows)}++// A merge is only correct when its comparator agrees with the order inside the+// streams it merges, and inside the array and object classes nothing derived+// from the values does: the service answers these three [1,2], [2], [1,3],+// while canonical JSON would put them [1,2], [1,3], [2] — an order that is+// neither, and that a TOP 2 would cut in the wrong place.+func TestOrderByPipeline_RefusesAnOrderItCannotMerge(t *testing.T) {+ p := newAscPipeline(t, "0", "1")++ res, err := p.Run()+ require.NoError(t, err)+ require.NotEmpty(t, res.Requests)+ feed := map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `[1,2]`, payload: `{"x":[1,2]}`},+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),+ "1": mkOrderByBody(t, ordered{ob: `[1,3]`, payload: `{"x":[1,3]}`}),+ }+ results := make([]queryengine.QueryResult, 0, len(res.Requests))+ for _, req := range res.Requests {+ results = append(results, queryengine.NewQueryResult(req.PartitionKeyRangeID, feed[req.PartitionKeyRangeID], ""))+ }+ require.NoError(t, p.ProvideData(results))++ _, err = p.Run()+ assert.ErrorIs(t, err, ErrUnsupportedPlanFeature)+}++// One array among scalars is still ordered, because the class ranking is the+// service's own: every array sorts after every string.+func TestOrderByPipeline_OrdersOneCompositeAmongScalars(t *testing.T) {+ p := newAscPipeline(t, "0", "1")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `"a"`, payload: `{"x":"a"}`},+ ordered{ob: `[9]`, payload: `{"x":[9]}`}),+ "1": mkOrderByBody(t, ordered{ob: `"b"`, payload: `{"x":"b"}`}),+ })+ assert.Equal(t, []string{`{"x":"a"}`, `{"x":"b"}`, `{"x":[9]}`}, rows)+}++// Rows past the window never reach the caller, so an order the engine cannot+// produce among them must not fail the query: TOP 2 over these streams is+// 0, 1, and the arrays behind them are nobody's business.+func TestOrderByPipeline_StopsAtTheLimitBeforeComparingExcludedRows(t *testing.T) {+ p := newTopOrderByPipeline(t, 2, "0", "1")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `0`, payload: `{"x":0}`},+ ordered{ob: `[1]`, payload: `{"x":[1]}`}),+ "1": mkOrderByBody(t,+ ordered{ob: `1`, payload: `{"x":1}`},+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),+ })+ assert.Equal(t, []string{`{"x":0}`, `{"x":1}`}, rows)+}++// TOP 0 asks for nothing, so no range is asked for a page and no row is ever+// compared.+func TestOrderByPipeline_AsksNothingOfRowsUnderTopZero(t *testing.T) {+ p := newTopOrderByPipeline(t, 0, "0", "1")++ res, err := p.Run()+ require.NoError(t, err)+ assert.Empty(t, res.Requests)+ assert.True(t, p.IsComplete())+}++// A page in flight when the window fills is dropped rather than merged, so an+// order the engine cannot produce inside it cannot fail a query that is already+// answered.+func TestOrderByPipeline_IgnoresAPageThatLandsAfterTheLimit(t *testing.T) {+ p := newTopOrderByPipeline(t, 1, "0", "1")++ res, err := p.Run()+ require.NoError(t, err)+ require.Len(t, res.Requests, 2)+ require.NoError(t, p.ProvideData([]queryengine.QueryResult{+ queryengine.NewQueryResult("0", mkOrderByBody(t, ordered{ob: `0`, payload: `{"x":0}`}), "more-0"),+ queryengine.NewQueryResult("1", mkOrderByBody(t, ordered{ob: `1`, payload: `{"x":1}`}), "more-1"),+ }))++ flushed, err := p.Run()+ require.NoError(t, err)+ got := make([]string, 0, len(flushed.Items))+ for _, item := range flushed.Items {+ got = append(got, string(item))+ }+ assert.Equal(t, []string{`{"x":0}`}, got)+ require.True(t, p.IsComplete())++ require.NoError(t, p.ProvideData([]queryengine.QueryResult{+ queryengine.NewQueryResult("0", mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1]}`}), ""),+ queryengine.NewQueryResult("1", mkOrderByBody(t, ordered{ob: `[2]`, payload: `{"x":[2]}`}), ""),+ }))++ // Nothing more may come out of it, and nothing in that page may fail it.+ late, err := p.Run()+ require.NoError(t, err)+ assert.Empty(t, late.Items)+ assert.True(t, p.IsComplete())+}++// Three ranges headed by a scalar and two arrays: the first row is the scalar+// whatever the arrays' order, so TOP 1 is answerable.+func TestOrderByPipeline_AnswersWhenTheFirstRowIsUnambiguous(t *testing.T) {+ p := newTopOrderByPipeline(t, 1, "0", "1", "2")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t, ordered{ob: `0`, payload: `{"x":0}`}),+ "1": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1]}`}),+ "2": mkOrderByBody(t, ordered{ob: `[2]`, payload: `{"x":[2]}`}),+ })+ assert.Equal(t, []string{`{"x":0}`}, rows)+}++// One range's own stream is already in the service's order, so two different+// arrays inside it are no trouble at all.+func TestOrderByPipeline_OrdersCompositesWithinOneRange(t *testing.T) {+ p := newAscPipeline(t, "0")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t,+ ordered{ob: `[1]`, payload: `{"x":[1]}`},+ ordered{ob: `[2]`, payload: `{"x":[2]}`}),+ })+ assert.Equal(t, []string{`{"x":[1]}`, `{"x":[2]}`}, rows)+}++// Two ranges holding the same value tie, and a tie is not an ambiguity.+func TestOrderByPipeline_OrdersEqualCompositesAcrossRanges(t *testing.T) {+ p := newAscPipeline(t, "0", "1")+ rows := runOnePass(t, p, map[string][]byte{+ "0": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1],"r":0}`}),+ "1": mkOrderByBody(t, ordered{ob: `[1]`, payload: `{"x":[1],"r":1}`}),+ })+ assert.Len(t, rows, 2)+}++// The ambiguity can surface mid-stream, once the scalars ahead of it are gone.+func TestOrderByPipeline_RefusesAnAmbiguityBehindTheScalars(t *testing.T) {+ p := newAscPipeline(t, "0", "1")++ res, err := p.Run()+ require.NoError(t, err)+ results := []queryengine.QueryResult{+ queryengine.NewQueryResult("0", mkOrderByBody(t,+ ordered{ob: `0`, payload: `{"x":0}`},+ ordered{ob: `[1,2]`, payload: `{"x":[1,2]}`}), ""),+ queryengine.NewQueryResult("1", mkOrderByBody(t,+ ordered{ob: `1`, payload: `{"x":1}`},+ ordered{ob: `[1,3]`, payload: `{"x":[1,3]}`}), ""),+ }+ require.Len(t, res.Requests, 2)+ require.NoError(t, p.ProvideData(results))++ var failure error+ for !p.IsComplete() {+ if _, err := p.Run(); err != nil {+ failure = err+ break+ }+ }+ assert.ErrorIs(t, failure, ErrUnsupportedPlanFeature)+}diff --git a/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go b/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.goindex ff2cb2d33b..524eee6a40 100644--- a/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go+++ b/backend/plugin/db/cosmosdb/queryengine/gonative/sentinel.go@@ -2,8 +2,18 @@ package gonativeimport "errors"-// ErrUnsupportedPlanFeature reports that the gateway's query plan needs a-// feature this engine does not implement, so the caller should surface the-// gateway's own error rather than the engine's. Upstream's queryengine package-// defines no such sentinel, so it lives with the engine that returns it.+// ErrUnsupportedPlanFeature reports that the engine cannot answer the query as+// asked. It comes back two ways, and the difference matters to the caller.+//+// From CreateQueryPipeline, before any row is read: the plan needs a feature+// the engine does not implement, and the driver can still surface the+// gateway's own error for the query.+//+// From a running pipeline, once the rows show what the plan did not: an ORDER+// BY over two values the service orders by its index's own encoding. Nothing+// is left to fall back to, so this one reaches the user as the engine's error+// — which is the point, since the alternative is a wrong order.+//+// Upstream's queryengine package defines no such sentinel, so it lives with+// the engine that returns it.var ErrUnsupportedPlanFeature = errors.New("cosmosdb/queryengine: unsupported plan feature")