Apache DataFusion SQL Query Engine
perf: PartitionedTopK hold retained rows in one shared RecordBatchStore (#25730)
Subham Singhal060ac929fd17
diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex f87a99944f..67f49d58bd 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -483,7 +483,9 @@ impl ExecutionPlan for PartitionedTopKExec {/// this operator instance.////// 2. **Emission** — `emit` drains all per-partition state in sorted-/// partition-key order, returning a coalesced batch stream. For+/// partition-key order. `ROW_NUMBER` interleaves the retained rows out+/// of its shared store in `batch_size` chunks, so its output needs no+/// coalescing; `RANK` and `DENSE_RANK` still coalesce. For/// `RANK`, boundary-tied rows are materialized and emitted after/// each partition's heap rows. For `DENSE_RANK`, rows are emitted/// from a K-bounded map of distinct ob keys, sorted ascending.@@ -492,7 +494,9 @@ impl ExecutionPlan for PartitionedTopKExec {////// - Time: O(N log K) where N = total rows, K = fetch/// - Memory: O(K × P × row_size) where P = number of distinct partitions-/// plus, for RANK, the boundary ties' rows+/// plus, for RANK, the boundary ties' rows. `ROW_NUMBER` holds its rows+/// by reference into gathered batches, so its constant is the store's+/// compaction ratio, and one in-flight gather is pinned on top.#[expect(clippy::too_many_arguments)]async fn do_partitioned_topk(partition_id: usize,diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rsindex dcb26b0b92..d4405c1f12 100644--- a/datafusion/physical-plan/src/topk/mod.rs+++ b/datafusion/physical-plan/src/topk/mod.rs@@ -43,7 +43,7 @@ use datafusion_common::{HashMap, Result, ScalarValue, internal_datafusion_err, internal_err,};use datafusion_execution::{- memory_pool::{MemoryConsumer, MemoryReservation},+ memory_pool::{MemoryConsumer, MemoryReservation, proxy::VecAllocExt},runtime_env::RuntimeEnv,};use datafusion_physical_expr::{@@ -884,6 +884,18 @@ impl TopKMetrics {}}+/// Rows a [`RecordBatchStore`] may hold per row still referenced by a heap,+/// before the referencing operator rewrites it.+///+/// A store entry is freed only when the last row referencing it is evicted, so+/// what it holds tracks the *input* rather than the rows retained. Both top-K+/// paths bound that the same way — compact once the store holds this multiple+/// of what is still referenced — and only the denominator differs: one heap's+/// length in [`TopKHeap::maybe_compact`], every partition's slots in+/// [`PartitionedTopK::compact_store`]. The multiplier avoids compacting when+/// the savings would be marginal.+const STORE_COMPACTION_RATIO: usize = 2;+/// This structure keeps at most the *smallest* k items, using the/// [arrow::row] format for sort keys. While it is called "topK" for/// values like `1, 2, 3, 4, 5` the "top 3" really means the@@ -1055,10 +1067,10 @@ impl TopKHeap {let total_rows = self.store.total_rows;let num_rows = self.inner.len();- // Compact when current store memory exceeds 2x what the compacted- // result would need. The multiplier avoids compacting when the+ // Compact when the store holds more than STORE_COMPACTION_RATIO x what+ // the compacted result would need. The multiplier avoids compacting when the// savings would be marginal.- if total_rows <= num_rows * 2 {+ if total_rows <= num_rows * STORE_COMPACTION_RATIO {return Ok(());}@@ -1207,6 +1219,16 @@ impl RecordBatchStore {RecordBatchEntry { id, batch, uses: 0 }}+ /// The id the next [`Self::register`] call will assign.+ ///+ /// For callers that must reference a batch before they can build it — see+ /// `PartitionedTopK::insert_batch`, which does not know which rows to gather+ /// until it has finished deciding. Valid until the next `register`, and only+ /// if nothing is left pointing at the id when that `register` is skipped.+ fn next_batch_id(&self) -> u32 {+ self.next_id+ }+/// Insert a record batch entry into this store, tracking its/// memory use, if it has any usespub fn insert(&mut self, entry: RecordBatchEntry) {@@ -1234,6 +1256,25 @@ impl RecordBatchStore {self.batches.len()}+ /// The stored batches in iteration order, paired with the+ /// `batch_id -> position` map that indexes them.+ ///+ /// `interleave_record_batch` addresses its inputs positionally while this+ /// store keys them by a non-contiguous id, so every caller that interleaves+ /// out of the store needs both. The batches are cloned — an `Arc` bump per+ /// column — so they can outlive the store: `emit` needs that, since the+ /// store is dropped as it returns. `compact_store` does not, and only+ /// shares the helper.+ fn positional(&self) -> (Vec<RecordBatch>, HashMap<u32, usize>) {+ let mut batches = Vec::with_capacity(self.batches.len());+ let mut positions = HashMap::with_capacity(self.batches.len());+ for (pos, (batch_id, entry)) in self.batches.iter().enumerate() {+ batches.push(entry.batch.clone());+ positions.insert(*batch_id, pos);+ }+ (batches, positions)+ }+/// returns true if the store has nothing storedfn is_empty(&self) -> bool {self.batches.is_empty()@@ -1274,45 +1315,224 @@ impl RecordBatchStore {}}+/// One retained row: its sort key by value, its output columns by reference.+///+/// `key` is the row-encoded ORDER BY tuple, owned inline. It is+/// byte-comparable, so `Ord` on the bytes is the sort order — the heap compares+/// rows without dereferencing anything, and eviction needs no access to the+/// payload at all.+///+/// `(batch_id, row)` locates the output columns in the operator's shared+/// [`RecordBatchStore`], which refcounts each batch by the slots pointing into+/// it. A slot's existence is what keeps its batch alive.+#[derive(Debug)]+struct PartitionSlot {+ key: Vec<u8>,+ /// Id in the shared store of the batch holding this row's output columns.+ batch_id: u32,+ /// Row of that batch.+ row: u32,+}++impl PartialEq for PartitionSlot {+ fn eq(&self, other: &Self) -> bool {+ self.key == other.key+ }+}+impl Eq for PartitionSlot {}+impl PartialOrd for PartitionSlot {+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {+ Some(self.cmp(other))+ }+}+impl Ord for PartitionSlot {+ fn cmp(&self, other: &Self) -> Ordering {+ self.key.cmp(&other.key)+ }+}++/// A single partition's top-K.+///+/// A max-heap on `key`, so the root is the *worst* retained row and a new row+/// only has to beat that one. Replacement mutates the root in place (reusing its+/// `Vec` capacity) and lets `PeekMut` sift down on drop — so a steady-state+/// eviction performs no allocation and no teardown.+///+/// There is one of these per *distinct partition key*, so every field is+/// multiplied by the partition count. Hence it is nothing but the heap, 24+/// bytes: the batches live in the operator's one shared [`RecordBatchStore`],+/// and `k` is a single constant for the whole operator, so it is passed to the+/// methods that need it rather than stored per partition.+#[derive(Debug, Default)]+struct PartitionHeap {+ inner: BinaryHeap<PartitionSlot>,+}++impl PartitionHeap {+ /// True if `key` belongs in the top-`k`.+ ///+ /// The worst retained row is the root, so this is one comparison — and+ /// while the heap is not yet full every row qualifies.+ fn qualifies(&self, k: usize, key: &[u8]) -> bool {+ if self.inner.len() < k {+ return true;+ }+ match self.inner.peek() {+ Some(worst) => key < worst.key.as_slice(),+ None => true,+ }+ }++ /// Retain `key`, pointing at row `row` of store batch `batch_id`, evicting+ /// the worst retained row if the heap is already full.+ ///+ /// Returns the evicted row's `batch_id`, which the caller owes the store an+ /// `unuse` for — this type does not touch the store itself, because the+ /// in-flight batch is not registered yet and only the caller knows that —+ /// along with the bytes this call newly allocated, which the operator folds+ /// into its running total. Returning the delta rather than keeping a running+ /// sum per heap is what lets this type hold nothing but the heap.+ fn add(+ &mut self,+ k: usize,+ key: &[u8],+ batch_id: u32,+ row: u32,+ ) -> (Option<u32>, usize) {+ debug_assert!(self.inner.len() <= k);+ if self.inner.len() == k {+ let mut worst = self.inner.peek_mut().expect("heap is full");+ let before = worst.key.capacity();+ worst.key.clear();+ worst.key.extend_from_slice(key);+ let grown = worst.key.capacity() - before;+ let evicted = worst.batch_id;+ worst.batch_id = batch_id;+ worst.row = row;+ drop(worst);+ (Some(evicted), grown)+ } else {+ let slot = PartitionSlot {+ key: key.to_vec(),+ batch_id,+ row,+ };+ let key_bytes = slot.key.capacity();+ let inner_before = self.inner.capacity();+ self.inner.push(slot);+ let grown = key_bytes+ + (self.inner.capacity() - inner_before) * size_of::<PartitionSlot>();+ (None, grown)+ }+ }++ /// The retained slots, in unspecified order.+ fn slots(&self) -> impl Iterator<Item = &PartitionSlot> + '_ {+ self.inner.iter()+ }++ /// Retained rows in ascending ORDER BY order, leaving this heap empty.+ fn drain_sorted(&mut self) -> Vec<PartitionSlot> {+ std::mem::take(&mut self.inner).into_sorted_vec()+ }++ /// Rewrite every slot's store coordinates, leaving the keys — and so the+ /// heap order — untouched.+ ///+ /// For [`PartitionedTopK::compact_store`], which moves every live row into+ /// a fresh set of store batches and has to repoint the slots at it.+ /// `into_vec` hands back the heap's own allocation and `BinaryHeap::from`+ /// takes it again, so this allocates nothing.+ fn repoint(&mut self, mut f: impl FnMut(&mut PartitionSlot)) {+ let mut slots = std::mem::take(&mut self.inner).into_vec();+ for slot in &mut slots {+ f(slot);+ }+ self.inner = BinaryHeap::from(slots);+ }+}+/// Top-K-per-partition operator state.////// Sibling to [`TopK`]. Where `TopK` maintains a single global heap,-/// `PartitionedTopK` maintains one [`TopKHeap`] per distinct partition-/// key while sharing a single [`RowConverter`], [`MemoryReservation`],-/// scratch [`Rows`] buffer, and [`TopKMetrics`] across all partitions.+/// `PartitionedTopK` maintains one [`PartitionHeap`] per distinct partition+/// key while sharing its two [`RowConverter`]s — one for the partition key,+/// one for the ORDER BY key — along with the [`MemoryReservation`], the+/// scratch [`Rows`] buffers, the [`RecordBatchStore`] holding retained rows,+/// and [`TopKMetrics`] across all partitions.////// This sharing is the point of the type: with N distinct partition/// keys, a naive `HashMap<_, TopK>` pays N × constant overhead for-/// `RowConverter::new`, `MemoryConsumer::register`, and metric-/// counter setup. `PartitionedTopK` pays it once.+/// `RowConverter::new`, `MemoryConsumer::register`, `RecordBatchStore::new`,+/// and metric counter setup. `PartitionedTopK` pays it once.pub(crate) struct PartitionedTopK {schema: SchemaRef,metrics: TopKMetrics,reservation: MemoryReservation,/// ORDER BY expressions (excludes PARTITION BY).expr: LexOrdering,- /// Encoder for ORDER BY columns. Reused across partitions.+ /// Encoder for the ORDER BY columns, whose encoding is the heap key.row_converter: RowConverter,- /// Scratch row buffer reused across `insert_batch` calls.+ /// Scratch row buffer for the ORDER BY encoding, reused across+ /// `insert_batch` calls.scratch_rows: Rows,/// PARTITION BY expressions.partition_exprs: Vec<Arc<dyn PhysicalExpr>>,- /// Encoder for the partition key.+ /// Encoder for the partition key. The encoding is byte-comparable with+ /// ASC/DESC and NULLS FIRST/LAST folded in, so the encoded bytes sort+ /// identically to the partition ordering — which is what lets `emit`+ /// recover partition-key order by sorting the keys directly.partition_converter: RowConverter,/// Scratch row buffer for partition-key encoding. Reused across/// `insert_batch` calls (cleared + appended each batch).partition_scratch_rows: Rows,- /// One heap per distinct partition key seen so far. Keyed by the- /// row-encoded PARTITION BY bytes (a byte-comparable encoding, so the- /// `Vec<u8>` hashes, compares, and sorts identically to an- /// `OwnedRow`) which lets `insert_batch` look partitions up with- /// `entry_ref` — allocating a key only on first sight of a partition- /// rather than once per row.- heaps: HashMap<Vec<u8>, TopKHeap>,- /// Scratch map reused across `insert_batch` calls to group a batch's- /// row indices by partition key. Drained (not reallocated) each batch- /// so its backing table is allocated once, not per batch.- partition_groups: HashMap<Vec<u8>, Vec<u32>>,+ /// One heap per distinct partition key seen so far, keyed by the+ /// row-encoded PARTITION BY key.+ ///+ /// `entry_ref` owns the key only on Vacant, so a key is allocated once per+ /// partition for the lifetime of the operator — not once per row, and not+ /// once per partition per batch (which is what draining a per-batch map+ /// would cost). Map order is arbitrary, so `emit` sorts the keys itself.+ heaps: HashMap<Vec<u8>, PartitionHeap>,+ /// The batches holding every retained row's output columns, refcounted by+ /// the slots pointing into them.+ ///+ /// One store for the whole operator rather than one per partition, which is+ /// what [`TopKHeap`] would give. Each entry holds only the rows that were+ /// admitted from one input batch — see `insert_batch` phase 3 — and is+ /// dropped as soon as the last slot referencing it is evicted. When that+ /// alone leaves it holding far more than the heaps point at,+ /// `compact_store` rewrites it.+ store: RecordBatchStore,+ /// Rows of the batch currently being inserted that were admitted, in+ /// ascending row order. Reused across `insert_batch` calls.+ admitted_rows: Vec<u32>,+ /// Rows the heaps currently hold, i.e. slots pointing into `store`.+ ///+ /// Tracked incrementally for the same reason as `heaps_bytes`: it is the+ /// denominator of `compact_store`'s ratio, read on every batch, and summing+ /// `inner.len()` over the heaps would be O(partitions seen so far). Slots+ /// are replaced rather than removed, so this only grows — it settles at+ /// `partitions × K` once every partition's heap has filled.+ live_slots: usize,+ /// Running sum of the bytes every [`PartitionHeap`] has allocated: each+ /// slot's key and the `BinaryHeap`'s own buffer.+ ///+ /// Maintained incrementally because `size()` runs on every batch: summing+ /// over the heaps would make it O(partitions seen so far) per batch, which+ /// is quadratic in the input whenever partition count grows with it. Every+ /// allocation it tracks only grows, so a running total stays exact without a+ /// decrement path. `PartitionHeap::add` returns each call's delta rather+ /// than keeping its own running sum, so the heaps stay free of a field that+ /// would be multiplied by the partition count.+ heaps_bytes: usize,+ /// Running sum of the partition keys `heaps` has interned, one per distinct+ /// key for the operator's lifetime.+ ///+ /// Counted by length rather than capacity: `entry_ref` builds the owned key+ /// with `to_vec`, so the two are equal. Separate from [`Self::heaps_bytes`]+ /// because it is the map's allocation, not any one heap's.+ index_bytes: usize,k: usize,batch_size: usize,}@@ -1334,6 +1554,9 @@ impl PartitionedTopK {let reservation = MemoryConsumer::new(format!("PartitionedTopK[{partition_id}]")).register(&runtime.memory_pool);+ // Both encoders are shared by every partition, and each scratch buffer+ // is sized to hold a whole batch so an `insert_batch` pass encodes once+ // per column set with no regrowth.let order_sort_fields = build_sort_fields(&order_expr, &schema)?;let row_converter = RowConverter::new(order_sort_fields)?;let scratch_rows =@@ -1354,15 +1577,19 @@ impl PartitionedTopK {partition_converter,partition_scratch_rows,heaps: HashMap::new(),- partition_groups: HashMap::new(),+ store: RecordBatchStore::new(),+ admitted_rows: Vec::new(),+ live_slots: 0,+ heaps_bytes: 0,+ index_bytes: 0,k,batch_size,})}- /// Demultiplex `batch` rows by partition key, encode the ORDER BY- /// columns once for the whole batch, and feed each partition's- /// rows into its dedicated [`TopKHeap`].+ /// Encode the partition and ORDER BY columns once each for the whole batch,+ /// admit every qualifying row into the [`PartitionHeap`] of the partition it+ /// belongs to, then register the admitted rows in the shared store.pub(crate) fn insert_batch(&mut self, batch: &RecordBatch) -> Result<()> {let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();let _timer = elapsed_compute.timer();@@ -1372,7 +1599,10 @@ impl PartitionedTopK {return Ok(());}- // 1. Evaluate + encode partition columns.+ // 1. Evaluate the partition and ORDER BY columns and encode each once+ // for the whole batch. Both encodes are whole-batch kernels that do+ // not care how the rows group, which is what lets the admission pass+ // below be a single row loop.let pk_arrays: Vec<ArrayRef> = self.partition_exprs.iter()@@ -1382,25 +1612,6 @@ impl PartitionedTopK {self.partition_converter.append(&mut self.partition_scratch_rows, &pk_arrays)?;- // 2. Demultiplex row indices by partition key (per-batch).- // `partition_groups` is a reused scratch map: taken out here and- // drained below, so its backing table is allocated once for the- // operator, not once per batch. `entry_ref` owns the key only on- // Vacant, so it allocates one `Vec<u8>` per distinct partition- // rather than one per row.- let mut groups = std::mem::take(&mut self.partition_groups);- groups.clear();- {- let pk_rows = &self.partition_scratch_rows;- for i in 0..num_rows {- groups- .entry_ref(pk_rows.row(i).as_ref())- .or_default()- .push(i as u32);- }- }-- // 3. Evaluate ORDER BY columns on the full batch and encode ONCE.let ob_arrays: Vec<ArrayRef> = self.expr.iter()@@ -1410,123 +1621,363 @@ impl PartitionedTopK {self.row_converter.append(&mut self.scratch_rows, &ob_arrays)?;- // 4. Per-partition: take the sub-batch, walk indices, dispatch- // qualifying rows into the partition's heap.+ // 2. One pass over the rows: find the row's partition and admit it to+ // that partition's heap if it qualifies. Only the key is copied+ // here; an admission records which row it kept and phase 3 gathers+ // them all in one `take`, so a batch is materialized once no matter+ // how many partitions it touched.+ //+ // Rows are visited in batch order rather than grouped by partition.+ // Interleaving groups cannot change a decision — a heap is only+ // read and written by rows of its own partition, which still reach+ // it in ascending row order — and it costs no pass over the+ // partitions seen so far, which would be quadratic in the input+ // whenever partition count grows with it.let k = self.k;let mut replacements: usize = 0;- for (pk, indices) in groups.drain() {- let heap = self.heaps.entry(pk).or_insert_with(|| TopKHeap::new(k));-- // Once a heap is full, most rows at high partition cardinality- // are rejected. Skip the gather + batch registration entirely- // when nothing in this partition group can improve the heap.- let any_qualify = indices.iter().any(|&orig_idx| {- let bytes = self.scratch_rows.row(orig_idx as usize);- match heap.max() {- Some(max_row) => bytes.as_ref() < max_row.row(),- None => true,+ self.admitted_rows.clear();+ // The gathered batch does not exist yet, but its id does, so a slot can+ // point at it during the pass with no back-patching afterwards. Its+ // `uses` is counted locally for the same reason.+ let batch_id = self.store.next_batch_id();+ let mut uses = 0usize;+ {+ let pk_rows = &self.partition_scratch_rows;+ let ob_rows = &self.scratch_rows;+ let heaps = &mut self.heaps;+ let admitted_rows = &mut self.admitted_rows;+ let store = &mut self.store;+ // Accumulated locally and folded in once, so the running totals are+ // not touched per row.+ let mut interned_bytes = 0usize;+ let mut admitted_bytes = 0usize;+ let mut new_slots = 0usize;++ for row in 0..num_rows {+ let pk = pk_rows.row(row);+ // One probe per row, and `entry_ref` owns the key only on+ // Vacant, so a key is allocated once per partition for the+ // operator's lifetime — not once per row.+ let mut interned = false;+ let heap = heaps.entry_ref(pk.as_ref()).or_insert_with(|| {+ interned = true;+ PartitionHeap::default()+ });+ if interned {+ // The map has just taken the only copy of this key.+ interned_bytes += pk.as_ref().len();}- });- if !any_qualify {- continue;- }-- let indices_arr = UInt32Array::from(indices);- let sub_batch = take_record_batch(batch, &indices_arr)?;- let mut entry = heap.register_batch(sub_batch);-- for (sub_idx, &orig_idx) in indices_arr.values().iter().enumerate() {- let row = self.scratch_rows.row(orig_idx as usize);- match heap.max() {- Some(max_row) if row.as_ref() >= max_row.row() => {}- None | Some(_) => {- heap.add(&mut entry, row, sub_idx);- replacements += 1;- }+ let key = ob_rows.row(row);+ if !heap.qualifies(k, key.as_ref()) {+ continue;}+ // An admission's row in the gathered batch is its position in+ // `admitted_rows`, because the gather preserves that order.+ let gather_pos = admitted_rows.len() as u32;+ uses += 1;+ let (evicted, grown) = heap.add(k, key.as_ref(), batch_id, gather_pos);+ admitted_bytes += grown;+ // Mirrors `TopKHeap::add`: a row evicted from the batch being+ // inserted is not in the store yet, so its use comes off the+ // local count rather than through `unuse`, which would panic on+ // an unregistered id.+ match evicted {+ Some(evicted_id) if evicted_id == batch_id => uses -= 1,+ Some(evicted_id) => store.unuse(evicted_id),+ // The heap was not yet full, so this slot is a new+ // reference into the store rather than a replaced one.+ None => new_slots += 1,+ }+ admitted_rows.push(row as u32);+ replacements += 1;}- heap.insert_batch_entry(entry);- heap.maybe_compact()?;+ self.index_bytes += interned_bytes;+ self.heaps_bytes += admitted_bytes;+ self.live_slots += new_slots;}- // Return the drained scratch map (capacity retained) for the next- // batch to reuse.- self.partition_groups = groups;+ // 3. Gather the rows this batch contributed into a single batch and hand+ // it to the store. Only rows admitted at some point during the pass+ // are kept, so what stays pinned is bounded by admissions rather than+ // by input size — and the entry is freed as soon as the last slot+ // referencing it is evicted.+ //+ // `uses == 0` means every admission from this batch was evicted again+ // before the pass ended, so there is nothing to keep and the id goes+ // back to the next batch.+ if uses > 0 {+ let gather_idx =+ UInt32Array::from_iter_values(self.admitted_rows.iter().copied());+ let mut entry = self.store.register(take_record_batch(batch, &gather_idx)?);+ debug_assert_eq!(+ entry.id, batch_id,+ "the id handed to the slots must be the id the gather got"+ );+ entry.uses = uses;+ self.store.insert(entry);+ }if replacements > 0 {self.metrics.row_replacements.add(replacements);}+ self.compact_store()?;self.reservation.try_resize(self.size())?;Ok(())}+ /// Rewrite the store to hold only the rows a heap still points at, once it+ /// holds [`STORE_COMPACTION_RATIO`]× more than that.+ ///+ /// An entry holds every row *admitted* from its input batch and is freed+ /// only when the last of them is evicted, so when survivors spread thinly+ /// one live row keeps a whole entry resident and residency tracks the+ /// *input*, not `partitions × K`: 512 partitions of `K = 1` fed 512 batches+ /// pin 131 K rows to retain 512. Nothing else bounds that. A single entry is+ /// no exception — rows admitted then superseded within their own batch stay+ /// in the gather unreferenced — so this does not skip a one-entry store.+ ///+ /// Amortized O(1) per admitted row: one pass over the live slots, and it+ /// cannot recur until the store has taken on another `live_slots` rows.+ ///+ /// Rows are rewritten into `batch_size` chunks, not one batch: an entry is+ /// released only when its last slot is evicted, so a single batch of every+ /// live row would free nothing until every partition has churned.+ ///+ /// Peak residency is the old store plus the new one — chunks are built+ /// before the old entries drop, and the reservation is not resized until+ /// `insert_batch` returns — so a pool sized at the steady-state bound can be+ /// exceeded transiently without erroring.+ ///+ /// All-or-nothing: plan the move, interleave, then rewrite the slots and the+ /// store. A failing interleave leaves the operator as it was rather than+ /// holding slots pointing at ids the store never got.+ fn compact_store(&mut self) -> Result<()> {+ if self.store.total_rows <= self.live_slots * STORE_COMPACTION_RATIO {+ return Ok(());+ }++ // These clones keep the old batches alive while the compacted ones are+ // built.+ let (old, array_pos) = self.store.positional();++ // Plan the move without touching anything: where each live row is now,+ // and where it is going. Keyed by the row's current `(batch_id, row)`+ // rather than by its position in this walk, so the rewrite below is free+ // to visit the heaps in any order — no two slots share a store row, so+ // the key identifies exactly one slot.+ let first_id = self.store.next_batch_id();+ let batch_size = self.batch_size;+ let mut coords: Vec<(usize, usize)> = Vec::with_capacity(self.live_slots);+ let mut moved: HashMap<(u32, u32), (u32, u32)> =+ HashMap::with_capacity(self.live_slots);+ for heap in self.heaps.values() {+ for slot in heap.slots() {+ let pos = *array_pos+ .get(&slot.batch_id)+ .expect("a live slot's batch_id is present in the store");+ let moved_to = coords.len();+ coords.push((pos, slot.row as usize));+ moved.insert(+ (slot.batch_id, slot.row),+ (+ first_id + (moved_to / batch_size) as u32,+ (moved_to % batch_size) as u32,+ ),+ );+ }+ }+ debug_assert_eq!(+ coords.len(),+ self.live_slots,+ "live_slots must count exactly the slots the heaps hold"+ );+ debug_assert_eq!(+ moved.len(),+ coords.len(),+ "two slots must not share a store row"+ );++ // The only fallible step, and nothing has been mutated yet: an error+ // here leaves both the heaps and the store exactly as they were.+ let refs: Vec<&RecordBatch> = old.iter().collect();+ let mut compacted: Vec<(RecordBatch, usize)> =+ Vec::with_capacity(coords.len().div_ceil(batch_size));+ for chunk in coords.chunks(batch_size) {+ compacted.push((interleave_record_batch(&refs, chunk)?, chunk.len()));+ }+ drop(refs);+ drop(old);++ // Infallible from here, so the store's ids and the slots agree again.+ for heap in self.heaps.values_mut() {+ heap.repoint(|slot| {+ let (batch_id, row) = moved[&(slot.batch_id, slot.row)];+ slot.batch_id = batch_id;+ slot.row = row;+ });+ }+ self.store.clear();+ for (chunk_idx, (batch, uses)) in compacted.into_iter().enumerate() {+ let mut entry = self.store.register(batch);+ debug_assert_eq!(+ entry.id,+ first_id + chunk_idx as u32,+ "the ids handed to the slots must be the ids the chunks got"+ );+ entry.uses = uses;+ self.store.insert(entry);+ }+ Ok(())+ }+/// Drain all heaps in partition-key order and return the rows as- /// a stream of coalesced `RecordBatch`es ordered by+ /// a stream of `RecordBatch`es ordered by/// `(partition_keys, order_keys)`.+ ///+ /// Retained rows live in the shared store as `(batch_id, row)` references,+ /// so this resolves them to `(array_pos, row)` pairs and interleaves them+ /// out in `batch_size` chunks. Because the chunking happens here the output+ /// batches are already the right size and no coalescing pass is needed.+ ///+ /// Only the ordering is done eagerly; the interleave happens one chunk per+ /// poll, which keeps `P × K` rows' worth of *output* arrays from existing at+ /// once. The store's batches stay pinned for the whole stream either way, so+ /// the reservation is carried into [`EmitState`] and released when the stream+ /// is dropped rather than when this returns.pub(crate) fn emit(self) -> Result<SendableRecordBatchStream> {let Self {schema,metrics,- reservation: _,+ reservation,expr: _,row_converter: _,scratch_rows: _,partition_exprs: _,partition_converter: _,partition_scratch_rows: _,- mut heaps,- partition_groups: _,+ heaps,+ store,+ admitted_rows: _,+ live_slots: _,+ heaps_bytes: _,+ index_bytes: _,k: _,batch_size,} = self;- let _timer = metrics.baseline.elapsed_compute().timer();-- let mut sorted_pks: Vec<Vec<u8>> = heaps.keys().cloned().collect();- sorted_pks.sort();-- let mut coalescer = BatchCoalescer::new(Arc::clone(&schema), batch_size);-- for pk in sorted_pks {- let mut heap = heaps.remove(&pk).expect("key from heaps.keys()");- if let Some(batch) = heap.emit()? {- coalescer.push_batch(batch)?;+ let timer = metrics.baseline.elapsed_compute().timer();++ // Map order is arbitrary, so partition-key order has to be recovered+ // explicitly here.+ let mut sorted_groups: Vec<(Vec<u8>, PartitionHeap)> =+ heaps.into_iter().collect();+ sorted_groups.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));++ // The batches outlive the store, which is dropped as this returns.+ let (batches, batch_id_array_pos) = store.positional();++ // Flattened in output order, so the emit itself is a slice walk.+ let mut ordered: Vec<(usize, usize)> = Vec::new();+ for (_key, mut heap) in sorted_groups {+ for slot in heap.drain_sorted() {+ let array_pos = *batch_id_array_pos+ .get(&slot.batch_id)+ .expect("a retained slot's batch_id is present in the store");+ ordered.push((array_pos, slot.row as usize));}}- coalescer.finish_buffered_batch()?;+ drop(timer);++ // What survives this function is the store's batches — pinned until the+ // returned stream is dropped — plus `ordered`, one 16-byte pair per+ // retained row. Everything else the operator held (both scratch+ // buffers, every heap and its interned key) is freed above, so the+ // resize below is normally a shrink. The reservation moves into the+ // stream state rather than being dropped here: releasing it while the+ // store is still pinned would stop accounting for bytes that are still+ // held, which is the one direction that matters.+ let pinned_bytes = store.batches_size;+ reservation.try_resize(+ size_of::<EmitState>()+ + pinned_bytes+ + batches.capacity() * size_of::<RecordBatch>()+ + ordered.capacity() * size_of::<(usize, usize)>(),+ )?;- let mut out: Vec<Result<RecordBatch>> = Vec::new();- while let Some(b) = coalescer.next_completed_batch() {- (&b).record_output(&metrics.baseline);- out.push(Ok(b));- }+ let state = EmitState {+ metrics,+ _reservation: reservation,+ batch_size,+ batches,+ ordered,+ pos: 0,+ };Ok(Box::pin(RecordBatchStreamAdapter::new(schema,- futures::stream::iter(out),+ futures::stream::try_unfold(state, |mut state| async move {+ Ok(state.next_batch()?.map(|batch| (batch, state)))+ }),)))}/// Total memory currently held by this operator, including all- /// per-partition heaps.+ /// per-partition heaps and every batch the store still pins.+ ///+ /// Every term is O(1): this runs on every batch, so the per-partition+ /// contributions are the running totals `heaps_bytes` and `index_bytes`+ /// rather than a sum over partitions.fn size(&self) -> usize {- // Per partition: the heap plus the encoded partition key owned by- // the map. The key bytes are a heap allocation the table slot- // doesn't cover.- let heaps_contents: usize = self- .heaps- .iter()- .map(|(pk, heap)| pk.capacity() + heap.size())- .sum();size_of::<Self>()+ self.row_converter.size()+ self.partition_converter.size()+ self.scratch_rows.size()+ self.partition_scratch_rows.size()- + heaps_contents- + self.heaps.capacity() * (size_of::<Vec<u8>>() + size_of::<TopKHeap>())- + self.partition_groups.capacity()- * (size_of::<Vec<u8>>() + size_of::<Vec<u32>>())+ + self.admitted_rows.allocated_size()+ + self.heaps.capacity() * (size_of::<Vec<u8>>() + size_of::<PartitionHeap>())+ + self.heaps_bytes+ + self.index_bytes+ + self.store.size()+ }+}++/// Hands out `batch_size` rows at a time, interleaved out of the store batches+/// the heaps referenced.+struct EmitState {+ metrics: TopKMetrics,+ /// Covers the pinned batches and `ordered` for as long as they are held.+ ///+ /// Carried here rather than dropped at the end of `emit` so the bytes stay+ /// accounted for until the stream is, and released by this struct's drop.+ /// Never resized: the store's batches are pinned until the last chunk, so+ /// there is nothing to hand back as chunks are emitted. Underscored because+ /// it is held only for that drop, as in `HashJoinStream` and `AsofJoinStream`.+ _reservation: MemoryReservation,+ batch_size: usize,+ /// The store's batches, positionally indexed by `ordered`.+ batches: Vec<RecordBatch>,+ /// `(array_pos, row)` pairs in `(partition_keys, order_keys)` order.+ ordered: Vec<(usize, usize)>,+ pos: usize,+}++impl EmitState {+ /// Build the next chunk, or `None` once every retained row has been+ /// emitted.+ fn next_batch(&mut self) -> Result<Option<RecordBatch>> {+ if self.pos == self.ordered.len() {+ return Ok(None);+ }+ let _timer = self.metrics.baseline.elapsed_compute().timer();++ let end = (self.pos + self.batch_size).min(self.ordered.len());+ let chunk = &self.ordered[self.pos..end];+ self.pos = end;++ let refs: Vec<&RecordBatch> = self.batches.iter().collect();+ let batch = interleave_record_batch(&refs, chunk)?;+ (&batch).record_output(&self.metrics.baseline);+ Ok(Some(batch))}}@@ -3396,6 +3847,29 @@ mod tests {k: usize,val_sort_options: SortOptions,val_nullable: bool,+ ) -> Result<(Arc<Schema>, PartitionedTopK)> {+ build_partitioned_topk_inner(+ k,+ val_sort_options,+ val_nullable,+ &Arc::new(RuntimeEnv::default()),+ )+ }++ /// Variant of [`build_partitioned_topk`] that runs against a caller-supplied+ /// [`RuntimeEnv`], so a test can bound the memory pool.+ fn build_partitioned_topk_with_runtime(+ k: usize,+ runtime: &Arc<RuntimeEnv>,+ ) -> Result<(Arc<Schema>, PartitionedTopK)> {+ build_partitioned_topk_inner(k, SortOptions::default(), false, runtime)+ }++ fn build_partitioned_topk_inner(+ k: usize,+ val_sort_options: SortOptions,+ val_nullable: bool,+ runtime: &Arc<RuntimeEnv>,) -> Result<(Arc<Schema>, PartitionedTopK)> {let schema = pk_val_schema(val_nullable);@@ -3409,77 +3883,449 @@ mod tests {options: val_sort_options,};- let partition_sort_fields = build_sort_fields(&[pk_sort_expr], &schema)?;+ let partition_ordering = vec![pk_sort_expr];let order_expr = LexOrdering::from([val_sort_expr]);let state = PartitionedTopK::try_new(0,Arc::clone(&schema),vec![pk_expr],- partition_sort_fields,+ build_sort_fields(&partition_ordering, &schema)?,order_expr,k,8, // batch_size- &Arc::new(RuntimeEnv::default()),+ runtime,&ExecutionPlanMetricsSet::new(),)?;Ok((schema, state))}- /// `PartitionedTopKRank` must not charge (or pin) a whole input batch- /// once per partition key that batch touches.- ///- /// Regression: the heap used to be handed `batch.clone()` — the full- /// input batch — rather than a gather of just that partition's rows,- /// so a batch spanning P partitions was counted P times over. With 500- /// partitions in one batch that reported ~516x the batch size, against- /// ~15x for the equivalent `ROW_NUMBER` state on identical input.- #[tokio::test]- async fn test_partitioned_topk_rank_size_is_not_per_partition_batch() -> Result<()> {- // The over-count factor was exactly the number of partitions sharing- // a batch, so P is what makes the bug visible at all. 500 turns the- // rank/row_number size ratio from ~1x into ~35x — far outside the- // range any accounting change could drift into.- const P: i32 = 500;- // 8 rows per partition: enough for every partition's k=2 heap to fill- // and then evict, so the heaps hold real state rather than sitting- // half-empty in the fill phase.- const ROWS: i32 = 4000;-- // Distinct values throughout: no ties, so RANK retains exactly what- // ROW_NUMBER does and the two sizes are directly comparable. k=2 is- // the smallest k with a distinct fill phase before eviction begins.- let pks: Vec<i32> = (0..ROWS).map(|i| i % P).collect();- let vals: Vec<i32> = (0..ROWS).map(|i| i / P).collect();-- let (schema, mut rn) = build_partitioned_topk(2)?;- rn.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;- let rn_size = rn.size();+ /// Partition-key nullability is a distinct concern from ORDER BY+ /// nullability: the key layer decides whether NULL keys collapse+ /// into one partition, and emit-time ordering decides where that+ /// partition lands relative to the non-NULL ones.+ fn build_partitioned_topk_nullable_pk(+ k: usize,+ pk_sort_options: SortOptions,+ ) -> Result<(Arc<Schema>, PartitionedTopK)> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("pk", DataType::Int32, true),+ Field::new("val", DataType::Int32, false),+ ]));- let (schema, mut rk) = build_partitioned_topk_rank(2)?;- rk.insert_batch(&pk_val_batch(&schema, pks, vals)?)?;- let rk_size = rk.size();+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;+ let partition_ordering = [PhysicalSortExpr {+ expr: Arc::clone(&pk_expr),+ options: pk_sort_options,+ }];+ let order_expr = LexOrdering::from([PhysicalSortExpr {+ expr: col("val", schema.as_ref())?,+ options: SortOptions::default(),+ }]);- // Measured ratio here is ~1.04 (18.1x vs 17.4x of the input batch);- // the bug produced ~35x. A 2x bar sits more than an order of magnitude- // clear of both, so this neither flakes when the size calculation is- // legitimately adjusted nor misses a reintroduced per-partition factor.- assert!(- rk_size <= rn_size * 2,- "RANK reported {rk_size} bytes for the same retained rows that \- ROW_NUMBER reported {rn_size} bytes for; a per-partition factor \- has crept back in"- );- Ok(())+ let state = PartitionedTopK::try_new(+ 0,+ Arc::clone(&schema),+ vec![pk_expr],+ build_sort_fields(&partition_ordering, &schema)?,+ order_expr,+ k,+ 8, // batch_size+ &Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),+ )?;+ Ok((schema, state))}- /// A boundary tie must retain only the tied rows, not the batch they- /// arrived in.- ///- /// Regression: `TieEntry` used to hold the source `RecordBatch` plus- /// indices, so one tied row pinned — and was charged for — a whole- /// input batch, for as long as the boundary held. Across a stream of- /// batches that made retained memory grow with the *input* size rather+ fn nullable_pk_batch(+ schema: &Arc<Schema>,+ pks: Vec<Option<i32>>,+ vals: Vec<i32>,+ ) -> Result<RecordBatch> {+ Ok(RecordBatch::try_new(+ Arc::clone(schema),+ vec![+ Arc::new(Int32Array::from(pks)),+ Arc::new(Int32Array::from(vals)),+ ],+ )?)+ }++ /// A NULL partition key is a partition like any other: every NULL-keyed+ /// row belongs to the *same* partition, and that partition gets its own+ /// K-row heap. With `NULLS LAST` on the partition key it emits after+ /// every non-NULL partition.+ #[tokio::test]+ async fn test_partitioned_topk_null_partition_key_nulls_last() -> Result<()> {+ let (schema, mut state) = build_partitioned_topk_nullable_pk(+ 2,+ SortOptions {+ descending: false,+ nulls_first: false,+ },+ )?;++ // pk=NULL vals: 9, 3, 5 → top-2 ASC = [3, 5] (one partition, not three)+ // pk=1 vals: 4, 2 → top-2 = [2, 4]+ // pk=2 vals: 8 → top-2 = [8]+ let batch = nullable_pk_batch(+ &schema,+ vec![None, Some(1), None, Some(2), None, Some(1)],+ vec![9, 4, 3, 8, 5, 2],+ )?;+ state.insert_batch(&batch)?;++ let results: Vec<_> = state.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+----+-----+",+ "| pk | val |",+ "+----+-----+",+ "| 1 | 2 |",+ "| 1 | 4 |",+ "| 2 | 8 |",+ "| | 3 |",+ "| | 5 |",+ "+----+-----+",+ ],+ &results+ );+ Ok(())+ }++ /// Companion to [`test_partitioned_topk_null_partition_key_nulls_last`]+ /// with `NULLS FIRST` on the partition key: the same NULL partition must+ /// now emit *before* every non-NULL one. Also splits the NULL key across+ /// two batches to prove it interns to a stable group.+ #[tokio::test]+ async fn test_partitioned_topk_null_partition_key_nulls_first() -> Result<()> {+ let (schema, mut state) = build_partitioned_topk_nullable_pk(+ 2,+ SortOptions {+ descending: false,+ nulls_first: true,+ },+ )?;++ // First batch sees pk=1 before any NULL, so the NULL partition is+ // interned second — emit must still place it first.+ state.insert_batch(&nullable_pk_batch(+ &schema,+ vec![Some(1), None, Some(1)],+ vec![4, 9, 2],+ )?)?;+ // Second batch must land in the *same* NULL partition and evict 9.+ state.insert_batch(&nullable_pk_batch(+ &schema,+ vec![None, None, Some(2)],+ vec![3, 5, 8],+ )?)?;++ let results: Vec<_> = state.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+----+-----+",+ "| pk | val |",+ "+----+-----+",+ "| | 3 |",+ "| | 5 |",+ "| 1 | 2 |",+ "| 1 | 4 |",+ "| 2 | 8 |",+ "+----+-----+",+ ],+ &results+ );+ Ok(())+ }++ /// A `DESC` partition key must emit partitions in descending key order.+ /// Group indices are assigned in first-seen order, so this pins the+ /// partition key's `SortOptions` really being folded into the encoding+ /// the emit-time sort reads, rather than defaulting to ascending.+ #[tokio::test]+ async fn test_partitioned_topk_desc_partition_key_ordering() -> Result<()> {+ let (schema, mut state) = build_partitioned_topk_nullable_pk(+ 1,+ SortOptions {+ descending: true,+ nulls_first: false,+ },+ )?;++ let batch = nullable_pk_batch(+ &schema,+ vec![Some(1), Some(3), Some(2), Some(3)],+ vec![10, 30, 20, 31],+ )?;+ state.insert_batch(&batch)?;++ let results: Vec<_> = state.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+----+-----+",+ "| pk | val |",+ "+----+-----+",+ "| 3 | 30 |",+ "| 2 | 20 |",+ "| 1 | 10 |",+ "+----+-----+",+ ],+ &results+ );+ Ok(())+ }++ /// A multi-column partition key is encoded as one concatenated byte+ /// string, so emit ordering becomes a lexicographic comparison of those+ /// bytes across both columns rather than of a single column's.+ #[tokio::test]+ async fn test_partitioned_topk_multi_column_partition_key() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("pk_a", DataType::Int32, false),+ Field::new("pk_b", DataType::Utf8, false),+ Field::new("val", DataType::Int32, false),+ ]));++ let pk_a: Arc<dyn PhysicalExpr> = col("pk_a", schema.as_ref())?;+ let pk_b: Arc<dyn PhysicalExpr> = col("pk_b", schema.as_ref())?;+ let partition_ordering = [+ PhysicalSortExpr {+ expr: Arc::clone(&pk_a),+ options: SortOptions::default(),+ },+ PhysicalSortExpr {+ expr: Arc::clone(&pk_b),+ options: SortOptions::default(),+ },+ ];+ let order_expr = LexOrdering::from([PhysicalSortExpr {+ expr: col("val", schema.as_ref())?,+ options: SortOptions::default(),+ }]);++ let mut state = PartitionedTopK::try_new(+ 0,+ Arc::clone(&schema),+ vec![pk_a, pk_b],+ build_sort_fields(&partition_ordering, &schema)?,+ order_expr,+ 1,+ 8,+ &Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),+ )?;++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![2, 1, 2, 1, 1])),+ Arc::new(StringArray::from(vec!["b", "b", "a", "a", "a"])),+ Arc::new(Int32Array::from(vec![20, 15, 25, 11, 10])),+ ],+ )?;+ state.insert_batch(&batch)?;++ let results: Vec<_> = state.emit()?.try_collect().await?;+ assert_batches_eq!(+ &[+ "+------+------+-----+",+ "| pk_a | pk_b | val |",+ "+------+------+-----+",+ "| 1 | a | 10 |",+ "| 1 | b | 15 |",+ "| 2 | a | 25 |",+ "| 2 | b | 20 |",+ "+------+------+-----+",+ ],+ &results+ );+ Ok(())+ }++ /// `PartitionedTopK` must run under a memory pool sized from `K` and the+ /// partition count, not from the input.+ ///+ /// Retained rows reference batches in the shared store, so what stays+ /// pinned is data-dependent in a way that `K` alone does not bound — this+ /// is the test that holds it down. A store entry keeps only the rows that+ /// batch contributed and is freed when its last row is evicted, so pinning+ /// converges instead of tracking the input; feeding 16 fully-admitted+ /// batches and asserting the total stays flat is what pins that.+ ///+ /// Sibling of `test_partitioned_topk_rank_runs_under_bounded_memory_pool`+ /// for the `ROW_NUMBER` path, which had no bounded-pool coverage.+ #[tokio::test]+ async fn test_partitioned_topk_runs_under_bounded_memory_pool() -> Result<()> {+ const P: i32 = 256;+ const ROWS_PER_PARTITION: i32 = 32;+ const ROWS: i32 = P * ROWS_PER_PARTITION;+ const BATCHES: i32 = 16;+ const K: usize = 2;++ let schema = pk_val_schema(false);+ let pks: Vec<i32> = (0..ROWS).map(|i| i % P).collect();+ // Batch b's values all beat batch b-1's, so every row of every batch is+ // admitted and every admission evicts. That is the maximum churn this+ // design can be put under -- a gather of the whole batch 16 times over,+ // every one of which must be handed back to the allocator as the next+ // batch supersedes it -- and it is the case where pinning that tracked+ // the input rather than converging would be obvious.+ let vals = |b: i32| -> Vec<i32> {+ (0..ROWS)+ .map(|i| (BATCHES - b) * ROWS_PER_PARTITION + i / P)+ .collect()+ };++ // Derived from the data so no byte constant here goes stale. The+ // operator needs a little over 4x the batch, most of it the two+ // row-encoding scratch buffers (partition key, ORDER BY key) sized to+ // hold a whole batch plus the one gathered batch the store pins; 4x+ // fails, so this bound is tight rather than slack. Charging a whole+ // batch per partition, as a per-partition store would, needs ~P = 256x.+ let batch_bytes =+ get_record_batch_memory_size(&pk_val_batch(&schema, pks.clone(), vals(0))?);+ let limit = 5 * batch_bytes;+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(limit)))+ .build_arc()?;++ let (schema, mut state) = build_partitioned_topk_with_runtime(K, &runtime)?;++ let mut after_first = 0;+ for b in 0..BATCHES {+ let batch = pk_val_batch(&schema, pks.clone(), vals(b))?;+ state.insert_batch(&batch).map_err(|e| {+ exec_datafusion_err!(+ "batch {b} of {BATCHES} failed under a {limit}-byte pool: {e}"+ )+ })?;+ if b == 0 {+ after_first = state.size();+ }+ }++ // Flat, not merely under the limit. Every batch was fully admitted, so+ // a store that failed to release a superseded entry would grow linearly+ // in the batch count here; the 2x slack absorbs hash-map and buffer+ // capacity growth but not a batch's worth of pinned rows.+ assert!(+ state.size() <= after_first * 2,+ "retention grew from {after_first} to {} bytes across {BATCHES} \+ batches; it tracks the input rather than K per partition",+ state.size()+ );++ // A pool that was never pressed proves nothing, so pin down the output:+ // K rows per partition, drawn from the last batch, whose values are the+ // smallest seen.+ let expected: Vec<(i32, i32)> = (0..P)+ .flat_map(|pk| [(pk, ROWS_PER_PARTITION), (pk, ROWS_PER_PARTITION + 1)])+ .collect();+ assert_eq!(pk_val_rows(state.emit()?).await?, expected);+ Ok(())+ }++ /// The bytes `emit` leaves pinned must stay accounted for until the stream+ /// is dropped.+ ///+ /// Retained rows reference batches in the shared store, so those batches are+ /// still held after `emit` returns, for as long as the stream lives. An+ /// `emit` that dropped its reservation instead would report zero while+ /// holding all of it — the one direction of accounting error that lets a+ /// pool overcommit. Asserting both sides catches the opposite mistake too:+ /// carrying the reservation but never releasing it.+ #[tokio::test]+ async fn test_partitioned_topk_emit_keeps_pinned_bytes_reserved() -> Result<()> {+ use datafusion_execution::memory_pool::MemoryPool;++ let pool = Arc::new(GreedyMemoryPool::new(16 * 1024 * 1024));+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::clone(&pool) as Arc<dyn MemoryPool>)+ .build_arc()?;++ let (schema, mut state) = build_partitioned_topk_with_runtime(2, &runtime)?;+ let pks: Vec<i32> = (0..64).map(|i| i % 8).collect();+ let vals: Vec<i32> = (0..64).collect();+ state.insert_batch(&pk_val_batch(&schema, pks, vals)?)?;+ assert!(pool.reserved() > 0, "insert_batch must reserve");++ let stream = state.emit()?;+ assert!(+ pool.reserved() > 0,+ "emit released its reservation while the store's batches are still \+ pinned by the stream it returned"+ );++ // Consumes the stream, so it is dropped by the time this returns.+ let results: Vec<RecordBatch> = stream.try_collect().await?;+ assert_eq!(+ pool.reserved(),+ 0,+ "the reservation must be released once the stream is dropped"+ );++ let rows: usize = results.iter().map(|b| b.num_rows()).sum();+ assert_eq!(rows, 16, "K = 2 for each of 8 partitions");+ Ok(())+ }++ /// `PartitionedTopKRank` must not charge (or pin) a whole input batch+ /// once per partition key that batch touches.+ ///+ /// Regression: the heap used to be handed `batch.clone()` — the full+ /// input batch — rather than a gather of just that partition's rows,+ /// so a batch spanning P partitions was counted P times over. With 500+ /// partitions in one batch that reported ~516x the batch size.+ ///+ /// Measured against the input batch's own size rather than against the+ /// `ROW_NUMBER` state on the same input: the two operators hold their+ /// retained rows differently, so a ratio between them tracks changes to+ /// either one and this test is about `RANK` alone.+ #[tokio::test]+ async fn test_partitioned_topk_rank_size_is_not_per_partition_batch() -> Result<()> {+ // The over-count factor was exactly the number of partitions sharing+ // a batch, so P is what makes the bug visible at all.+ const P: i32 = 500;+ // 8 rows per partition: enough for every partition's k=2 heap to fill+ // and then evict, so the heaps hold real state rather than sitting+ // half-empty in the fill phase.+ const ROWS: i32 = 4000;++ // Distinct values throughout: no ties, so RANK retains exactly K per+ // partition and nothing is held by the tie list. k=2 is the smallest k+ // with a distinct fill phase before eviction begins.+ let pks: Vec<i32> = (0..ROWS).map(|i| i % P).collect();+ let vals: Vec<i32> = (0..ROWS).map(|i| i / P).collect();++ let (schema, mut rk) = build_partitioned_topk_rank(2)?;+ let batch = pk_val_batch(&schema, pks, vals)?;+ let batch_size = get_record_batch_memory_size(&batch);+ rk.insert_batch(&batch)?;+ let rk_size = rk.size();++ // Just under 19x the input batch measured here, against ~516x for the bug:+ // the per-partition scratch buffers and heap headers at P=500 dominate+ // what is retained, which is why the correct figure is a double-digit+ // multiple at all. A 100x bar sits ~5x clear of both, so it neither+ // flakes when the size calculation is legitimately adjusted nor misses a+ // reintroduced per-partition batch charge.+ assert!(+ rk_size <= batch_size * 100,+ "RANK reported {rk_size} bytes holding K=2 rows from each of {P} \+ partitions of a {batch_size}-byte batch; a per-partition batch \+ charge has crept back in"+ );+ Ok(())+ }++ /// A boundary tie must retain only the tied rows, not the batch they+ /// arrived in.+ ///+ /// Regression: `TieEntry` used to hold the source `RecordBatch` plus+ /// indices, so one tied row pinned — and was charged for — a whole+ /// input batch, for as long as the boundary held. Across a stream of+ /// batches that made retained memory grow with the *input* size rather/// than with `K + ties`.#[tokio::test]async fn test_partitioned_topk_rank_ties_do_not_pin_input_batches() -> Result<()> {@@ -3678,6 +4524,38 @@ mod tests {.collect()}+ /// Brute-force reference for `ROW_NUMBER`: per partition, the `k`+ /// smallest values by the ORDER BY key.+ ///+ /// Deliberately not expressed through [`Self::expected`]. That ranks a+ /// row by how many rows precede it, so with `k = 2` and values+ /// `[5, 5, 5]` every row ranks 1 and a `<= k` filter keeps all three —+ /// `RANK` semantics, not `ROW_NUMBER`, which keeps exactly two.+ ///+ /// Comparing `(pk, val)` pairs makes *which* of a tied group is kept+ /// unobservable, so this reference stays deterministic despite+ /// `DiffShape` generating ties on purpose.+ ///+ /// Built in emit order rather than sorted at the end: the `BTreeMap`+ /// walks partitions ascending and each partition's values are sorted+ /// ascending, which is exactly `(partition_key, order_key)` for the+ /// ASC/ASC sort options `build_partitioned_topk` uses. That makes this+ /// usable with [`pk_val_rows`], so the differential test checks the+ /// emitted order and not just the set.+ fn expected_row_number(&self) -> Vec<(i32, i32)> {+ let mut by_pk: std::collections::BTreeMap<i32, Vec<i32>> =+ std::collections::BTreeMap::new();+ for (pk, val) in self.all_rows() {+ by_pk.entry(pk).or_default().push(val);+ }+ let mut kept: Vec<(i32, i32)> = Vec::new();+ for (pk, mut vals) in by_pk {+ vals.sort_unstable();+ kept.extend(vals.into_iter().take(self.k).map(|v| (pk, v)));+ }+ kept+ }+/// Brute-force reference: the sorted rows a `<= k` filter keeps, given/// `rank_of(all_rows, pk, val)` for the ranking function under test.fn expected(@@ -3695,15 +4573,13 @@ mod tests {}}- /// Drain an operator's output into sorted `(pk, val)` pairs, ready to- /// compare against [`DiffShape::expected`].+ /// Drain an operator's output into `(pk, val)` pairs **in emit order**.///- /// Asserts the rows arrived already in `(pk ASC, val ASC)` order before- /// sorting them. That order is load-bearing rather than cosmetic — it is- /// what `PartitionedTopKExec::compute_properties` advertises, and so what- /// lets the window above it run `mode=Sorted` with no `SortExec` — and- /// the content comparison this feeds would otherwise sort it away.- async fn sorted_pk_val(stream: SendableRecordBatchStream) -> Result<Vec<(i32, i32)>> {+ /// Preferred over [`sorted_pk_val`] wherever the expected rows can be+ /// written in `(partition_key, order_key)` order, because emitting them in+ /// the wrong order is a real failure this then catches. Sorting here would+ /// hide it: a reversed `emit` passes a sorted comparison on every shape.+ async fn pk_val_rows(stream: SendableRecordBatchStream) -> Result<Vec<(i32, i32)>> {let batches: Vec<RecordBatch> = stream.try_collect().await?;let mut rows: Vec<(i32, i32)> = Vec::new();for b in &batches {@@ -3713,6 +4589,19 @@ mod tests {rows.push((pk.value(i), val.value(i)));}}+ Ok(rows)+ }++ /// [`pk_val_rows`] sorted, ready to compare against+ /// [`DiffShape::expected`].+ ///+ /// Asserts the rows arrived already in `(pk ASC, val ASC)` order before+ /// sorting them. That order is load-bearing rather than cosmetic — it is+ /// what `PartitionedTopKExec::compute_properties` advertises, and so what+ /// lets the window above it run `mode=Sorted` with no `SortExec` — and+ /// the content comparison this feeds would otherwise sort it away.+ async fn sorted_pk_val(stream: SendableRecordBatchStream) -> Result<Vec<(i32, i32)>> {+ let mut rows = pk_val_rows(stream).await?;assert!(rows.windows(2).all(|w| w[0] <= w[1]),"emitted rows are not in (pk, val) order: {rows:?}"@@ -3721,6 +4610,603 @@ mod tests {Ok(rows)}+ /// Randomized differential test for the plain `ROW_NUMBER` path.+ ///+ /// `PartitionedTopKRank` and `PartitionedTopKDenseRank` each had one of+ /// these; `ROW_NUMBER` did not, which left its retained-row storage+ /// without a randomized net.+ ///+ /// Compares in emit order, via [`pk_val_rows`] rather than+ /// [`sorted_pk_val`]: `(partition_key, order_key)` ordering is the+ /// operator's output contract, and a sorted comparison cannot see it —+ /// reversing both the partition sort and the within-partition order in+ /// `emit` leaves a sorted version of this test green on all 64 seeds.+ #[tokio::test]+ async fn test_partitioned_topk_matches_bruteforce() -> Result<()> {+ for seed in 0..64u64 {+ let shape = DiffShape::new(seed, 6);+ let (schema, mut state) = build_partitioned_topk(shape.k)?;+ for (pks, vals) in &shape.batches {+ state.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;+ }++ assert_eq!(+ pk_val_rows(state.emit()?).await?,+ shape.expected_row_number(),+ "{shape}"+ );+ }+ Ok(())+ }++ /// A row can be admitted and then superseded by a better row from the+ /// *same* batch, before that batch has been registered in the store.+ ///+ /// This is the case the use count has to get right: the superseded row's+ /// use is dropped on the in-flight count rather than through `unuse`, which+ /// would panic on an id the store has never seen. Every such row still ends+ /// up in the gather, so the surviving slot's row index has to account for+ /// admissions that no longer survive.+ ///+ /// Interleaving two partitions is what makes a slot pointing at the wrong+ /// gathered row visible: crossing them yields `(0, 2)` / `(1, 1)` instead+ /// of `(0, 1)` / `(1, 2)`, which a single-partition case could not tell+ /// apart from correct behaviour.+ #[tokio::test]+ async fn test_partitioned_topk_payload_survives_supersession() -> Result<()> {+ // pk 0 admits 9, then 3 supersedes it, then 1 supersedes that.+ // pk 1 admits 8, then 2 supersedes it.+ let (schema, mut state) = build_partitioned_topk(1)?;+ state.insert_batch(&pk_val_batch(+ &schema,+ vec![0, 1, 0, 1, 0],+ vec![9, 8, 3, 2, 1],+ )?)?;++ assert_eq!(pk_val_rows(state.emit()?).await?, vec![(0, 1), (1, 2)]);++ // With K = 2 a supersession replaces only the worst slot, so the+ // other retained row must keep pointing at its own gathered row.+ let (schema, mut state) = build_partitioned_topk(2)?;+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0, 0], vec![4, 7, 5])?)?;++ assert_eq!(pk_val_rows(state.emit()?).await?, vec![(0, 4), (0, 5)]);+ Ok(())+ }++ /// The store must release a gathered batch the moment its last referencing+ /// slot is evicted, and must never register a batch nothing references.+ ///+ /// This is the property that makes pinning converge rather than track the+ /// input. The bounded-pool test above asserts the *consequence* (total size+ /// stays flat); this asserts the mechanism directly, because a leak that+ /// happened to be offset by a shrinking buffer elsewhere would satisfy a+ /// total-size bound while still holding batches forever.+ #[tokio::test]+ async fn test_partitioned_topk_store_releases_superseded_batches() -> Result<()> {+ const K: usize = 2;+ let (schema, mut state) = build_partitioned_topk(K)?;++ // Batch 0 fills the single partition's heap, so it is referenced and+ // must be pinned.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0], vec![50, 60])?)?;+ assert_eq!(state.store.len(), 1, "the batch every slot points at");+ let first_id = *state.store.batches.keys().next().expect("one entry");++ // Batch 1 is entirely worse, so nothing is admitted and nothing may be+ // registered — a store that registered unconditionally would grow here.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0], vec![70, 80])?)?;+ assert_eq!(+ state.store.len(),+ 1,+ "a fully-rejected batch must not be registered"+ );++ // Batch 2 supersedes both retained rows, so batch 0 loses its last+ // reference and must be dropped as batch 2 takes its place.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0], vec![10, 20])?)?;+ assert_eq!(+ state.store.len(),+ 1,+ "superseded batch 0 must be released, not accumulated"+ );+ assert!(+ !state.store.batches.contains_key(&first_id),+ "the released entry must be the superseded one"+ );++ // Partial supersession: one of the two retained rows is replaced, so+ // both the old and the new batch are legitimately referenced.+ state.insert_batch(&pk_val_batch(&schema, vec![0], vec![5])?)?;+ assert_eq!(+ state.store.len(),+ 2,+ "a batch still holding one live row must stay pinned"+ );++ assert_eq!(pk_val_rows(state.emit()?).await?, vec![(0, 5), (0, 10)]);+ Ok(())+ }++ /// Pinned rows must stay proportional to the rows the heaps hold, even when+ /// releasing superseded batches alone cannot manage it.+ ///+ /// The sibling test above covers the easy shape: every batch supersedes the+ /// last one whole, so every entry's use count reaches zero and the store+ /// converges to one entry on its own. This is the shape where it does not.+ /// `val = |batch - partition|` makes batch `j` the owner of partition `j`'s+ /// winner, so all `B` entries keep exactly one live row and each stays+ /// resident holding every row it admitted. Releasing is powerless here —+ /// nothing is ever fully superseded — and without `compact_store` the store+ /// pins `B(B+1)/2` rows (131,328 at `B = 512`, half the whole input) to+ /// retain `B`. That is the unbounded case: over-retention grows with `B`.+ #[tokio::test]+ async fn test_partitioned_topk_store_compacts_when_survivors_spread() -> Result<()> {+ const B: i32 = 512;++ let (schema, mut state) = build_partitioned_topk(1)?;+ for j in 0..B {+ let pks: Vec<i32> = (0..B).collect();+ let vals: Vec<i32> = (0..B).map(|p| (j - p).abs()).collect();+ state.insert_batch(&pk_val_batch(&schema, pks, vals)?)?;+ }++ // Every partition holds its K = 1 row, so the ratio's denominator is B.+ assert_eq!(state.live_slots, B as usize);+ let pinned: usize = state+ .store+ .batches+ .values()+ .map(|e| e.batch.num_rows())+ .sum();+ assert_eq!(pinned, state.store.total_rows, "store row count is exact");+ // An invariant, not a measurement: `compact_store` runs at the end of+ // every `insert_batch`, so on return the store either never tripped the+ // guard or was just rewritten down to `live_slots`. Expressed through+ // the constant so tuning it cannot leave this stale. Measured 977 here+ // against 131,328 unfixed, so the bar bites by two orders of magnitude+ // at this B and the gap widens with it.+ let bound = B as usize * STORE_COMPACTION_RATIO;+ assert!(+ pinned <= bound,+ "{pinned} rows pinned to retain {} — above the {bound}-row bound, so \+ retention tracks the input rather than the rows held",+ state.live_slots+ );++ // Compaction rewrote every slot's coordinates, so the rows it resolves+ // at emit are the check that it repointed them correctly: partition p's+ // single retained row is the one whose value is 0, contributed by batch+ // p — a slot left pointing at a pre-compaction coordinate would surface+ // some other partition's row here. Compared in emit order, so this also+ // covers the ordering of a compacted, multi-chunk emit; no other test+ // reaches that path at all.+ let expected: Vec<(i32, i32)> = (0..B).map(|pk| (pk, 0)).collect();+ assert_eq!(pk_val_rows(state.emit()?).await?, expected);+ Ok(())+ }++ /// A store holding a *single* entry must still be compacted when that entry+ /// is mostly dead rows.+ ///+ /// Regression: `compact_store` used to bail on `store.len() <= 1`, on the+ /// reasoning that one entry cannot compact to anything smaller. It can. An+ /// entry holds every row *admitted* from its batch, and a row admitted and+ /// then superseded within that same pass stays in the gather with nothing+ /// pointing at it — so a single entry is bounded by the input batch, not by+ /// `live_slots`.+ ///+ /// Descending values in one batch is the worst case and not an exotic one:+ /// every row beats the current worst, so all of them are admitted and `K`+ /// survive. `store.len() == 1` is also the state right after every+ /// compaction and on the first batch, so the old guard was live for the+ /// whole of a single-batch query.+ #[tokio::test]+ async fn test_partitioned_topk_store_compacts_a_single_oversized_entry() -> Result<()>+ {+ const ROWS: i32 = 200;++ let (schema, mut state) = build_partitioned_topk(1)?;+ state.insert_batch(&pk_val_batch(+ &schema,+ vec![0; ROWS as usize],+ (0..ROWS).map(|i| ROWS - i).collect(),+ )?)?;++ assert_eq!(state.live_slots, 1, "K = 1 in a single partition");+ assert_eq!(state.store.len(), 1, "one input batch, one gathered entry");+ // The bound the ratio promises, which the old guard exempted this shape+ // from entirely: 200 rows stayed pinned to retain 1.+ assert!(+ state.store.total_rows <= state.live_slots * STORE_COMPACTION_RATIO,+ "{} rows pinned to retain {}; a single entry escaped compaction",+ state.store.total_rows,+ state.live_slots+ );++ // Compaction repointed the surviving slot, so emit still resolves it.+ assert_eq!(pk_val_rows(state.emit()?).await?, vec![(0, 1)]);+ Ok(())+ }++ /// Every invariant the shared store's bookkeeping rests on, checked against+ /// a full recount of the heaps.+ ///+ /// The running totals (`live_slots`, `store.total_rows`,+ /// `store.batches_size`, each entry's `uses`) are maintained incrementally+ /// precisely so that neither `size()` nor `compact_store`'s guard has to+ /// walk the partitions. Nothing else re-derives them, so a drift in any one+ /// is invisible: an over-counted `uses` leaks an entry forever, an+ /// under-counted one drops a batch rows still point at, and a wrong+ /// `live_slots` silently disables the pinning bound.+ fn assert_store_invariants(state: &PartitionedTopK, label: &str) {+ // 1. `live_slots` is the number of slots the heaps actually hold.+ let counted: usize = state.heaps.values().map(|h| h.inner.len()).sum();+ assert_eq!(+ counted, state.live_slots,+ "{label}: live_slots disagrees with the heaps"+ );++ // 2. Every slot points at a live entry, and each entry's `uses` is+ // exactly the number of slots pointing into it.+ let mut refs: HashMap<u32, usize> = HashMap::new();+ for heap in state.heaps.values() {+ for slot in heap.slots() {+ assert!(+ state.store.get(slot.batch_id).is_some(),+ "{label}: slot points at batch {} which the store does not hold",+ slot.batch_id+ );+ *refs.entry(slot.batch_id).or_default() += 1;+ }+ }+ assert_eq!(+ refs.len(),+ state.store.len(),+ "{label}: the store holds entries nothing points at"+ );+ for (id, entry) in &state.store.batches {+ assert_eq!(+ entry.uses,+ refs.get(id).copied().unwrap_or(0),+ "{label}: entry {id} has uses={} but {} slots point at it",+ entry.uses,+ refs.get(id).copied().unwrap_or(0)+ );+ // A slot's row must be in range, or emit's interleave would read+ // out of bounds (or silently pick a different row).+ for heap in state.heaps.values() {+ for slot in heap.slots().filter(|s| s.batch_id == *id) {+ assert!(+ (slot.row as usize) < entry.batch.num_rows(),+ "{label}: slot row {} is outside entry {id} ({} rows)",+ slot.row,+ entry.batch.num_rows()+ );+ }+ }+ }++ // 3. The store's two running totals are exact, not estimates.+ let rows: usize = state+ .store+ .batches+ .values()+ .map(|e| e.batch.num_rows())+ .sum();+ assert_eq!(+ rows, state.store.total_rows,+ "{label}: store.total_rows drifted"+ );+ let bytes: usize = state+ .store+ .batches+ .values()+ .map(|e| get_record_batch_memory_size(&e.batch))+ .sum();+ assert_eq!(+ bytes, state.store.batches_size,+ "{label}: store.batches_size drifted"+ );++ // 4. The bound `compact_store` exists to enforce. Without it, what stays+ // pinned tracks the *input* rather than `partitions × K`.+ assert!(+ state.store.total_rows <= state.live_slots * STORE_COMPACTION_RATIO,+ "{label}: {} rows pinned to retain {}",+ state.store.total_rows,+ state.live_slots+ );++ // 5. What the operator reported to the pool is what it computes now.+ assert_eq!(+ state.reservation.size(),+ state.size(),+ "{label}: the reservation does not match size()"+ );+ }++ /// Feed shapes designed to break the store's bookkeeping, checking every+ /// invariant after every batch and the output against brute force at the+ /// end.+ ///+ /// The shapes matter more than the count. `K = 1` with one partition per+ /// batch is the sparse-survivor case that makes a live row pin a whole+ /// gathered entry; a descending run at `K = 1` admits every row and keeps+ /// one, so the gather is nearly all dead on arrival and the pre-assigned+ /// batch id churns down to `uses == 1`; an all-ties batch admits nothing+ /// after the first `k` rows, so `uses == 0` and the id must go back to the+ /// next batch unconsumed.+ #[tokio::test]+ async fn test_partitioned_topk_store_bookkeeping_holds_under_stress() -> Result<()> {+ use rand::rngs::StdRng;+ use rand::{Rng, SeedableRng};++ struct Case {+ name: &'static str,+ k: usize,+ partitions: i32,+ values: i32,+ batches: usize,+ rows: usize,+ /// Sort each batch's values descending, so every row is an+ /// admission that the next row supersedes.+ descending: bool,+ /// Give each batch a single partition, so survivors spread one per+ /// gathered entry.+ one_partition_per_batch: bool,+ }++ let cases = [+ Case {+ name: "k1_one_partition_per_batch",+ k: 1,+ partitions: 64,+ values: 1_000,+ batches: 64,+ rows: 16,+ descending: false,+ one_partition_per_batch: true,+ },+ Case {+ name: "k1_descending_runs",+ k: 1,+ partitions: 3,+ values: 1_000,+ batches: 20,+ rows: 50,+ descending: true,+ one_partition_per_batch: false,+ },+ Case {+ name: "all_ties_uses_zero",+ k: 2,+ partitions: 1,+ values: 1,+ batches: 20,+ rows: 30,+ descending: false,+ one_partition_per_batch: false,+ },+ Case {+ name: "k_exceeds_partition_size",+ k: 40,+ partitions: 50,+ values: 1_000,+ batches: 30,+ rows: 10,+ descending: false,+ one_partition_per_batch: false,+ },+ Case {+ name: "wide_churn",+ k: 3,+ partitions: 12,+ values: 20,+ batches: 40,+ rows: 64,+ descending: false,+ one_partition_per_batch: false,+ },+ ];++ for case in cases {+ for seed in 0..4u64 {+ let mut rng = StdRng::seed_from_u64(seed);+ let (schema, mut state) = build_partitioned_topk(case.k)?;+ let mut all_rows: Vec<(i32, i32)> = Vec::new();++ for b in 0..case.batches {+ let pks: Vec<i32> = if case.one_partition_per_batch {+ vec![(b as i32) % case.partitions; case.rows]+ } else {+ (0..case.rows)+ .map(|_| rng.random_range(0..case.partitions))+ .collect()+ };+ let mut vals: Vec<i32> = (0..case.rows)+ .map(|_| rng.random_range(0..case.values))+ .collect();+ if case.descending {+ vals.sort_unstable_by(|a, b| b.cmp(a));+ }+ all_rows.extend(pks.iter().copied().zip(vals.iter().copied()));++ state.insert_batch(&pk_val_batch(&schema, pks, vals)?)?;+ assert_store_invariants(+ &state,+ &format!("{} seed {seed} batch {b}", case.name),+ );+ }++ // Brute force: per partition, the k smallest values, emitted in+ // (pk, val) ascending order.+ let mut expected: Vec<(i32, i32)> = Vec::new();+ let mut pks: Vec<i32> = all_rows.iter().map(|&(p, _)| p).collect();+ pks.sort_unstable();+ pks.dedup();+ for pk in pks {+ let mut vals: Vec<i32> = all_rows+ .iter()+ .filter(|&&(p, _)| p == pk)+ .map(|&(_, v)| v)+ .collect();+ vals.sort_unstable();+ vals.truncate(case.k);+ expected.extend(vals.into_iter().map(|v| (pk, v)));+ }++ assert_eq!(+ pk_val_rows(state.emit()?).await?,+ expected,+ "{} seed {seed}",+ case.name+ );+ }+ }+ Ok(())+ }++ /// A batch whose every admission is superseded before the pass ends must+ /// not consume the batch id `next_batch_id` handed the slots.+ ///+ /// `insert_batch` points slots at an id before the gather exists, and skips+ /// `register` entirely when `uses` falls back to 0. If the id were consumed+ /// anyway the sequence would gap harmlessly, but if `register` ran with+ /// `uses == 0` the store would keep an entry nothing references — and+ /// `compact_store`'s `first_id` would then collide with it.+ #[tokio::test]+ async fn test_partitioned_topk_unused_gather_id_is_reused() -> Result<()> {+ let (schema, mut state) = build_partitioned_topk(1)?;++ // Batch 0: one partition, one row. Consumes id 0.+ state.insert_batch(&pk_val_batch(&schema, vec![0], vec![5])?)?;+ assert_eq!(state.store.next_batch_id(), 1, "batch 0 consumed its id");++ // Batch 1: every value is worse than the retained 5, so nothing is+ // admitted at all — `uses` never rises and the id is untouched. No+ // compaction either: neither total_rows nor live_slots moved, and the+ // guard already held when the previous call returned.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0, 0], vec![7, 8, 9])?)?;+ assert_eq!(state.store.next_batch_id(), 1, "no admission, no id");+ assert_eq!(state.store.len(), 1);+ assert_store_invariants(&state, "after a batch with no admissions");++ // Batch 2: rows are admitted but each is superseded by a later row of+ // the *same* batch, and the first evicts the row from batch 0. The+ // gather therefore holds 3 rows for 1 live slot, which trips+ // `compact_store` — so this consumes two ids, the gather's and the+ // compacted chunk's.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0, 0], vec![4, 3, 2])?)?;+ assert_store_invariants(&state, "after a self-superseding batch");+ assert_eq!(state.store.len(), 1, "batch 0's entry was released");+ assert_eq!(+ state.store.total_rows, 1,+ "compaction dropped the dead rows"+ );+ let after_compaction = state.store.next_batch_id();++ // Batch 3: admitted then superseded within the batch, ending worse than+ // the retained 2 — so `uses` returns to 0 and the id is handed back.+ state.insert_batch(&pk_val_batch(&schema, vec![0, 0], vec![3, 6])?)?;+ assert_eq!(+ state.store.next_batch_id(),+ after_compaction,+ "every admission was superseded, so the id must be reusable"+ );+ assert_store_invariants(&state, "after a fully superseded batch");++ assert_eq!(pk_val_rows(state.emit()?).await?, vec![(0, 2)]);+ Ok(())+ }++ /// `size()` must account for the ORDER BY keys the heaps hold by value.+ ///+ /// Every other term of `size()` is either a fixed struct, a `RowConverter`,+ /// or the store — all of which exist whatever the key width. `heaps_bytes`+ /// is the only term that grows with `partitions × K × key_width`, and with+ /// an `Int32` key it is a rounding error, so a test on the narrow schema+ /// cannot see it go missing. With a wide string key it is the dominant+ /// per-partition term: dropping the `heaps_bytes` fold leaves the operator+ /// under-reporting megabytes to the pool.+ #[tokio::test]+ async fn test_partitioned_topk_size_accounts_for_retained_keys() -> Result<()> {+ const PARTITIONS: usize = 200;+ const K: usize = 4;+ const KEY_WIDTH: usize = 512;++ let schema = Arc::new(Schema::new(vec![+ Field::new("pk", DataType::Int32, false),+ Field::new("val", DataType::Utf8, false),+ ]));+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;+ let partition_ordering = [PhysicalSortExpr {+ expr: Arc::clone(&pk_expr),+ options: SortOptions::default(),+ }];+ let order_expr = LexOrdering::from([PhysicalSortExpr {+ expr: col("val", schema.as_ref())?,+ options: SortOptions::default(),+ }]);+ let mut state = PartitionedTopK::try_new(+ 0,+ Arc::clone(&schema),+ vec![pk_expr],+ build_sort_fields(&partition_ordering, &schema)?,+ order_expr,+ K,+ 8,+ &Arc::new(RuntimeEnv::default()),+ &ExecutionPlanMetricsSet::new(),+ )?;++ let empty = state.size();++ // Feed K rows for each partition, all retained, each key KEY_WIDTH wide.+ let mut pks: Vec<i32> = Vec::new();+ let mut vals: Vec<String> = Vec::new();+ for p in 0..PARTITIONS {+ for i in 0..K {+ pks.push(p as i32);+ vals.push(format!("{p:0>width$}{i}", width = KEY_WIDTH - 1));+ }+ }+ state.insert_batch(&RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(pks)),+ Arc::new(StringArray::from(vals)),+ ],+ )?)?;++ assert_eq!(+ state.live_slots,+ PARTITIONS * K,+ "every row should have been retained"+ );++ // The keys alone are this many bytes; `size()` has to have grown by at+ // least that much, on top of whatever the store and the map cost.+ let key_bytes = PARTITIONS * K * KEY_WIDTH;+ assert!(+ state.heaps_bytes >= key_bytes,+ "heaps_bytes is {} but the retained keys are at least {key_bytes} bytes",+ state.heaps_bytes+ );+ assert!(+ state.size() >= empty + key_bytes,+ "size() grew by {} for {key_bytes} bytes of retained keys",+ state.size() - empty+ );+ // And the interned partition keys are counted separately.+ assert!(+ state.index_bytes >= PARTITIONS,+ "index_bytes is {} for {PARTITIONS} interned keys",+ state.index_bytes+ );+ Ok(())+ }+/// Randomized differential test for `PartitionedTopKRank`.////// The retention rule is subtle — a K-bounded heap plus a boundary-tie@@ -3856,8 +5342,8 @@ mod tests {/// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for/// the partitioned emit path: rows from several per-partition heaps are- /// coalesced into `batch_size` batches, so the metric must count the- /// coalesced output rather than the per-partition inputs.+ /// interleaved into `batch_size` chunks, so the metric must count the+ /// emitted chunks rather than the per-partition heaps they came from.#[tokio::test]async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches()-> Result<()> {@@ -3866,24 +5352,22 @@ mod tests {Field::new("val", DataType::Int32, false),]));let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;- let partition_sort_fields = build_sort_fields(- &[PhysicalSortExpr {- expr: Arc::clone(&pk_expr),- options: SortOptions::default(),- }],- &schema,- )?;+ let partition_ordering = vec![PhysicalSortExpr {+ expr: Arc::clone(&pk_expr),+ options: SortOptions::default(),+ }];let metrics = ExecutionPlanMetricsSet::new();- // 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3+ // 3 per-partition heaps of 2 rows each, emitted as 2 batches of 3+ let order_ordering = [PhysicalSortExpr {+ expr: col("val", schema.as_ref())?,+ options: SortOptions::default(),+ }];let mut state = PartitionedTopK::try_new(0,Arc::clone(&schema),vec![pk_expr],- partition_sort_fields,- LexOrdering::from([PhysicalSortExpr {- expr: col("val", schema.as_ref())?,- options: SortOptions::default(),- }]),+ build_sort_fields(&partition_ordering, &schema)?,+ LexOrdering::from(order_ordering.clone()),2,3, // batch_size&Arc::new(RuntimeEnv::default()),