Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

7.1 KB
Raw Blame
package gonative

import (
	"encoding/json"

	"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos/queryengine"
	"github.com/pkg/errors"
)

// distinctPartitionResponse decodes the per-partition response body for a
// DISTINCT query. Unlike aggregate pipelines, the per-partition Documents is
// an array of raw JSON values: one projected object per object-form DISTINCT,
// or one scalar per DISTINCT VALUE. We preserve them as json.RawMessage so the
// de-duplication stage can hash the canonical form without type-specific code.
type distinctPartitionResponse struct {
	Documents []json.RawMessage `json:"Documents"`
}

// parseDistinctPartitionResponse decodes a DISTINCT per-partition body.
func parseDistinctPartitionResponse(raw []byte) (*distinctPartitionResponse, error) {
	var r distinctPartitionResponse
	if err := json.Unmarshal(raw, &r); err != nil {
		return nil, errors.Wrap(err, "gonative: parse distinct partition response")
	}
	return &r, nil
}

// canonicalJSON re-marshals the given JSON value so that semantically equal
// values produce byte-equal output (sorted object keys, no extraneous whitespace).
// Returns the canonical bytes so the caller can use them as a map key.
//
// The approach: unmarshal into `any`, which turns objects into map[string]any,
// then re-marshal — Go's encoding/json sorts map keys alphabetically in that
// path, which gives us the canonical ordering for free.
func canonicalJSON(raw json.RawMessage) ([]byte, error) {
	var v any
	if err := json.Unmarshal(raw, &v); err != nil {
		return nil, errors.Wrap(err, "gonative: canonicalize")
	}
	return json.Marshal(v)
}

// distinctPipeline implements queryengine.QueryPipeline for unordered DISTINCT
// queries. It streams raw documents in the order partitions deliver them and
// emits each unique value the first time it is seen. The pipeline works
// uniformly for object DISTINCT (SELECT DISTINCT c.field FROM c) and
// DISTINCT VALUE (SELECT DISTINCT VALUE c.field FROM c) because both forms
// land in the Documents array as raw JSON values.
//
// The seen-set grows unbounded — same as the .NET SDK's
// Microsoft.Azure.Cosmos.UnorderedDistinctMap (see
// https://github.com/Azure/azure-cosmos-dotnet-v3/blob/master/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/Distinct/DistinctMap.UnorderedDistinctMap.cs).
// The OS is the memory ceiling. Queries whose DISTINCT cardinality could
// exhaust memory should scope themselves with a WHERE clause.
type distinctPipeline struct {
	query  string
	fanout rangeFanout
	closed bool

	// offset and limit are the plan's global skip and take, applied after
	// dedup: the gateway strips OFFSET/LIMIT and TOP from the rewritten query
	// so that each range reports its own distinct values in full and the page
	// is cut once, over the whole result. A negative limit means unlimited.
	offset  int
	limit   int
	skipped int
	taken   int

	// Pending items not yet returned to the caller. Populated during
	// ProvideData as each partition's response is de-duplicated against the
	// seen set.
	pending [][]byte

	// seen is the global hash-set of canonicalized documents. Keys are the
	// canonical JSON bytes rendered as a string — Go strings are immutable
	// and comparable, so they make lightweight map keys even for long JSON.
	seen map[string]struct{}
}

// newDistinctPipeline builds a pipeline for a DISTINCT plan. "Ordered" and
// "Unordered" differ only in whether the gateway promises sorted input, which
// a global seen-set does not need; any other distinctType is a shape the
// engine has not seen and rejects.
func newDistinctPipeline(plan *planDoc, pkRangeIDs []string, query string) (*distinctPipeline, error) {
	if plan.QueryInfo.DistinctType != "Unordered" && plan.QueryInfo.DistinctType != "Ordered" {
		return nil, ErrUnsupportedPlanFeature
	}
	offset, limit, err := planSkipTake(plan.QueryInfo)
	if err != nil {
		return nil, err
	}
	// A plain DISTINCT plan carries no rewrittenQuery, so each range answers
	// the user's own query. A DISTINCT the user paged or capped does carry
	// one — the same DISTINCT with the OFFSET/LIMIT or TOP taken off — and
	// running that instead is what lets this pipeline cut the page globally.
	if plan.QueryInfo.RewrittenQuery != "" {
		query = plan.QueryInfo.RewrittenQuery
	}
	return &distinctPipeline{
		query:  query,
		fanout: newRangeFanout(pkRangeIDs),
		offset: offset,
		limit:  limit,
		seen:   make(map[string]struct{}),
	}, nil
}

// Query implements queryengine.QueryPipeline.
func (p *distinctPipeline) Query() string { return p.query }

// IsComplete implements queryengine.QueryPipeline. A DISTINCT pipeline is
// complete once every range has delivered its last page and every deduped
// item has been flushed to the caller.
func (p *distinctPipeline) IsComplete() bool {
	if len(p.pending) > 0 {
		return false
	}
	if p.limit >= 0 && p.taken >= p.limit {
		// LIMIT satisfied — whatever the ranges still hold cannot change the
		// answer, so the pipeline stops asking for it.
		return true
	}
	return p.fanout.allDrained()
}

// Close implements queryengine.QueryPipeline.
func (p *distinctPipeline) Close() { p.closed = true }

// Run drives one pipeline turn. Deduplicated items go out as soon as they are
// known, so a DISTINCT over many pages does not hold the whole result before
// the caller sees its first row:
//  1. Flush any pending deduplicated items.
//  2. Otherwise ask for the next page of every range that has one — the
//     initial fan-out on the first call.
func (p *distinctPipeline) Run() (*queryengine.PipelineResult, error) {
	if len(p.pending) > 0 {
		batch := p.pending
		p.pending = nil
		return &queryengine.PipelineResult{
			Items:       batch,
			IsCompleted: p.IsComplete(),
		}, nil
	}
	if p.IsComplete() {
		return &queryengine.PipelineResult{IsCompleted: true}, nil
	}
	if reqs := p.fanout.requests(p.query); len(reqs) > 0 {
		return &queryengine.PipelineResult{Requests: reqs}, nil
	}
	return &queryengine.PipelineResult{IsCompleted: p.IsComplete()}, nil
}

// ProvideData implements queryengine.QueryPipeline. Each incoming partition
// response is parsed and its documents are fed through the global seen set.
// First-sightings pass to OFFSET/LIMIT and then become pending items;
// duplicates are silently dropped.
func (p *distinctPipeline) ProvideData(results []queryengine.QueryResult) error {
	for _, r := range results {
		resp, err := parseDistinctPartitionResponse(r.Data)
		if err != nil {
			return err
		}
		for _, doc := range resp.Documents {
			canonical, err := canonicalJSON(doc)
			if err != nil {
				return err
			}
			key := string(canonical)
			if _, ok := p.seen[key]; ok {
				continue
			}
			p.seen[key] = struct{}{}
			if p.skipped < p.offset {
				p.skipped++
				continue
			}
			if p.limit >= 0 && p.taken >= p.limit {
				break
			}
			// Preserve the partition's original document bytes in the output —
			// callers receive what the gateway returned, not our canonicalized
			// form. Copy defensively: json.RawMessage is aliased to the input
			// buffer, which the SDK may reuse for later pages.
			p.pending = append(p.pending, append([]byte(nil), doc...))
			p.taken++
		}
		if err := p.fanout.note(r); err != nil {
			return err
		}
	}
	return nil
}