Skip to content

Apache DataFusion SQL Query Engine

fix: reduce RANK window top-K memory from O(input) to O(K + ties) per partition (#24591)

Jay Zhane1eaeb18e4e8
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index 940033857d..5fd64389d4 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -23,7 +23,7 @@ use arrow::{
BatchCoalescer, FilterBuilder, interleave_record_batch, prep_null_mask_filter,
take_record_batch,
},
- row::{OwnedRow, RowConverter, Rows, SortField},
+ row::{RowConverter, Rows, SortField},
};
use datafusion_expr::{ColumnarValue, Operator};
use std::mem::size_of;
@@ -1273,8 +1273,20 @@ pub(crate) struct PartitionedTopK {
partition_exprs: Vec<Arc<dyn PhysicalExpr>>,
/// Encoder for the partition key.
partition_converter: RowConverter,
- /// One heap per distinct partition key seen so far.
- heaps: HashMap<OwnedRow, TopKHeap>,
+ /// Scratch row buffer for partition-key encoding. Reused across
+ /// `insert_batch` calls (cleared + appended each batch).
+ partition_scratch_rows: Rows,
+ /// One heap per distinct partition key seen so far. Keyed by the
+ /// row-encoded PARTITION BY bytes (a byte-comparable encoding, so the
+ /// `Vec<u8>` hashes, compares, and sorts identically to an
+ /// `OwnedRow`) which lets `insert_batch` look partitions up with
+ /// `entry_ref` — allocating a key only on first sight of a partition
+ /// rather than once per row.
+ heaps: HashMap<Vec<u8>, TopKHeap>,
+ /// Scratch map reused across `insert_batch` calls to group a batch's
+ /// row indices by partition key. Drained (not reallocated) each batch
+ /// so its backing table is allocated once, not per batch.
+ partition_groups: HashMap<Vec<u8>, Vec<u32>>,
k: usize,
batch_size: usize,
}
@@ -1302,6 +1314,8 @@ impl PartitionedTopK {
row_converter.empty_rows(batch_size, ESTIMATED_BYTES_PER_ROW * batch_size);
let partition_converter = RowConverter::new(partition_sort_fields)?;
+ let partition_scratch_rows = partition_converter
+ .empty_rows(batch_size, ESTIMATED_BYTES_PER_ROW * batch_size);
Ok(Self {
schema,
@@ -1312,7 +1326,9 @@ impl PartitionedTopK {
scratch_rows,
partition_exprs,
partition_converter,
+ partition_scratch_rows,
heaps: HashMap::new(),
+ partition_groups: HashMap::new(),
k,
batch_size,
})
@@ -1336,15 +1352,26 @@ impl PartitionedTopK {
.iter()
.map(|e| e.evaluate(batch).and_then(|v| v.into_array(num_rows)))
.collect::<Result<_>>()?;
- let pk_rows = self.partition_converter.convert_columns(&pk_arrays)?;
+ self.partition_scratch_rows.clear();
+ self.partition_converter
+ .append(&mut self.partition_scratch_rows, &pk_arrays)?;
// 2. Demultiplex row indices by partition key (per-batch).
- let mut groups: HashMap<OwnedRow, Vec<u32>> = HashMap::new();
- for i in 0..num_rows {
- groups
- .entry(pk_rows.row(i).owned())
- .or_default()
- .push(i as u32);
+ // `partition_groups` is a reused scratch map: taken out here and
+ // drained below, so its backing table is allocated once for the
+ // operator, not once per batch. `entry_ref` owns the key only on
+ // Vacant, so it allocates one `Vec<u8>` per distinct partition
+ // rather than one per row.
+ let mut groups = std::mem::take(&mut self.partition_groups);
+ groups.clear();
+ {
+ let pk_rows = &self.partition_scratch_rows;
+ for i in 0..num_rows {
+ groups
+ .entry_ref(pk_rows.row(i).as_ref())
+ .or_default()
+ .push(i as u32);
+ }
}
// 3. Evaluate ORDER BY columns on the full batch and encode ONCE.
@@ -1361,7 +1388,7 @@ impl PartitionedTopK {
// qualifying rows into the partition's heap.
let k = self.k;
let mut replacements: usize = 0;
- for (pk, indices) in groups {
+ for (pk, indices) in groups.drain() {
let heap = self.heaps.entry(pk).or_insert_with(|| TopKHeap::new(k));
// Once a heap is full, most rows at high partition cardinality
@@ -1397,6 +1424,10 @@ impl PartitionedTopK {
heap.maybe_compact()?;
}
+ // Return the drained scratch map (capacity retained) for the next
+ // batch to reuse.
+ self.partition_groups = groups;
+
if replacements > 0 {
self.metrics.row_replacements.add(replacements);
}
@@ -1417,13 +1448,15 @@ impl PartitionedTopK {
scratch_rows: _,
partition_exprs: _,
partition_converter: _,
+ partition_scratch_rows: _,
mut heaps,
+ partition_groups: _,
k: _,
batch_size,
} = self;
let _timer = metrics.baseline.elapsed_compute().timer();
- let mut sorted_pks: Vec<OwnedRow> = heaps.keys().cloned().collect();
+ let mut sorted_pks: Vec<Vec<u8>> = heaps.keys().cloned().collect();
sorted_pks.sort();
let mut coalescer = BatchCoalescer::new(Arc::clone(&schema), batch_size);
@@ -1451,24 +1484,39 @@ impl PartitionedTopK {
/// Total memory currently held by this operator, including all
/// per-partition heaps.
fn size(&self) -> usize {
+ // Per partition: the heap plus the encoded partition key owned by
+ // the map. The key bytes are a heap allocation the table slot
+ // doesn't cover.
+ let heaps_contents: usize = self
+ .heaps
+ .iter()
+ .map(|(pk, heap)| pk.capacity() + heap.size())
+ .sum();
size_of::<Self>()
+ self.row_converter.size()
+ self.partition_converter.size()
+ self.scratch_rows.size()
- + self.heaps.values().map(|h| h.size()).sum::<usize>()
- + self.heaps.capacity() * (size_of::<OwnedRow>() + size_of::<TopKHeap>())
+ + self.partition_scratch_rows.size()
+ + heaps_contents
+ + self.heaps.capacity() * (size_of::<Vec<u8>>() + size_of::<TopKHeap>())
+ + self.partition_groups.capacity()
+ * (size_of::<Vec<u8>>() + size_of::<Vec<u32>>())
}
}
-/// A run of rows from a single source [`RecordBatch`] that tied at the
-/// boundary when inserted. Stored as `(batch, indices)` and materialized
-/// at emit time via [`take_record_batch`].
+/// Rows that tied at the boundary when inserted, materialized into a
+/// batch holding *only* those rows.
+///
+/// The rows are gathered eagerly rather than kept as `(source_batch,
+/// indices)`: a tie entry lives until the boundary moves, so holding the
+/// source batch would pin an entire input batch — and charge for it —
+/// for as long as a single row of it stays tied. With ties spread across
+/// many input batches that makes retained memory grow with the *input*
+/// size instead of with `K + ties`.
#[derive(Debug)]
struct TieEntry {
+ /// The tied rows, and nothing else. Always non-empty by construction.
batch: RecordBatch,
- /// Indices into `batch` of the rows tied at the (then-current)
- /// boundary. Always non-empty by construction.
- row_indices: Vec<u32>,
/// `get_record_batch_memory_size(&batch)` captured at push time so
/// `RankPartitionState::size()` doesn't recurse through `batch`'s
/// columns on every `try_resize` call.
@@ -1490,13 +1538,15 @@ struct RankPartitionState {
impl RankPartitionState {
fn size(&self) -> usize {
let ties_buffer = self.ties.capacity() * size_of::<TieEntry>();
- let ties_contents: usize = self
- .ties
- .iter()
- .map(|t| t.row_indices.capacity() * size_of::<u32>() + t.batch_bytes)
- .sum();
+ let ties_contents: usize = self.ties.iter().map(|t| t.batch_bytes).sum();
self.heap.size() + ties_buffer + ties_contents
}
+
+ /// Push `batch`'s rows onto the tie list, charging exactly their bytes.
+ fn push_ties(&mut self, batch: RecordBatch) {
+ let batch_bytes = get_record_batch_memory_size(&batch);
+ self.ties.push(TieEntry { batch, batch_bytes });
+ }
}
/// Sibling to [`PartitionedTopK`] implementing `RANK()` semantics.
@@ -1541,8 +1591,17 @@ pub(crate) struct PartitionedTopKRank {
/// `insert_batch` calls (cleared + appended each batch) so we
/// avoid allocating a fresh `Rows` buffer every batch.
partition_scratch_rows: Rows,
- /// One rank state per distinct partition key seen so far.
- states: HashMap<OwnedRow, RankPartitionState>,
+ /// One rank state per distinct partition key seen so far. Keyed by
+ /// the row-encoded PARTITION BY bytes (a byte-comparable encoding, so
+ /// the `Vec<u8>` hashes, compares, and sorts identically to an
+ /// `OwnedRow`) which lets `insert_batch` look partitions up with
+ /// `entry_ref` — allocating a key only on first sight of a partition
+ /// rather than once per row.
+ states: HashMap<Vec<u8>, RankPartitionState>,
+ /// Scratch map reused across `insert_batch` calls to group a batch's
+ /// row indices by partition key. Drained (not reallocated) each batch
+ /// so its backing table is allocated once, not per batch.
+ partition_groups: HashMap<Vec<u8>, Vec<u32>>,
k: usize,
batch_size: usize,
}
@@ -1585,6 +1644,7 @@ impl PartitionedTopKRank {
partition_converter,
partition_scratch_rows,
states: HashMap::new(),
+ partition_groups: HashMap::new(),
k,
batch_size,
})
@@ -1603,12 +1663,6 @@ impl PartitionedTopKRank {
return Ok(());
}
- // Captured once so the per-tie push from this batch can reuse
- // it (computing `get_record_batch_memory_size` is O(cols ×
- // buffer walk) and we'd otherwise pay it per push and again
- // per `try_resize` call).
- let input_batch_bytes = get_record_batch_memory_size(batch);
-
// 1. Evaluate + encode partition columns into the reusable
// scratch (cleared then appended).
let pk_arrays: Vec<ArrayRef> = self
@@ -1619,15 +1673,23 @@ impl PartitionedTopKRank {
self.partition_scratch_rows.clear();
self.partition_converter
.append(&mut self.partition_scratch_rows, &pk_arrays)?;
- let pk_rows = &self.partition_scratch_rows;
// 2. Demultiplex row indices by partition key (per-batch).
- let mut groups: HashMap<OwnedRow, Vec<u32>> = HashMap::new();
- for i in 0..num_rows {
- groups
- .entry(pk_rows.row(i).owned())
- .or_default()
- .push(i as u32);
+ // `partition_groups` is a reused scratch map: taken out here and
+ // drained below, so its backing table is allocated once for the
+ // operator, not once per batch. `entry_ref` owns the key only on
+ // Vacant, so it allocates one `Vec<u8>` per distinct partition
+ // rather than one per row.
+ let mut groups = std::mem::take(&mut self.partition_groups);
+ groups.clear();
+ {
+ let pk_rows = &self.partition_scratch_rows;
+ for i in 0..num_rows {
+ groups
+ .entry_ref(pk_rows.row(i).as_ref())
+ .or_default()
+ .push(i as u32);
+ }
}
// 3. Evaluate ORDER BY columns on the full batch and encode ONCE.
@@ -1644,22 +1706,45 @@ impl PartitionedTopKRank {
let k = self.k;
let mut replacements: usize = 0;
- for (pk, indices) in groups {
+ for (pk, indices) in groups.drain() {
let state = self.states.entry(pk).or_insert_with(|| RankPartitionState {
heap: TopKHeap::new(k),
ties: Vec::new(),
});
- // Equal indices for THIS batch only. Coalesced into a single
- // tie entry at the end of the partition's loop. Discarded if
- // the boundary moves up mid-loop (those rows were tied to the
- // old boundary, which is now strictly worse than the new K-th).
+ // Once the heap is full, a group whose rows are *all* strictly
+ // worse than the boundary changes neither the heap nor the
+ // ties. Bail before the gather below — at high partition
+ // cardinality this is the common case.
+ if let Some(max_row) = state.heap.max() {
+ let boundary = max_row.row();
+ if indices
+ .iter()
+ .all(|&i| self.scratch_rows.row(i as usize).as_ref() > boundary)
+ {
+ continue;
+ }
+ }
+
+ // Gather this partition's rows into their own batch, as
+ // `PartitionedTopK` does. Registering the whole input batch
+ // instead would pin it — and charge for it — once per
+ // partition key present in the batch, so a batch spanning P
+ // partitions would be counted P times over.
+ let indices_arr = UInt32Array::from(indices);
+ let sub_batch = take_record_batch(batch, &indices_arr)?;
+
+ // Indices *into `sub_batch`* of rows from this batch that tied
+ // at the boundary. Coalesced into a single tie entry at the end
+ // of the partition's loop. Discarded if the boundary moves up
+ // mid-loop (those rows were tied to the old boundary, which is
+ // now strictly worse than the new K-th).
let mut equal_indices: Vec<u32> = Vec::new();
// Lazy-registered: only attached if at least one row reaches
// the heap from this batch in this partition.
- let mut entry: Option<RecordBatchEntry> = None;
+ let mut heap_entry: Option<RecordBatchEntry> = None;
- for &orig_idx in &indices {
+ for (sub_idx, &orig_idx) in indices_arr.values().iter().enumerate() {
let row = self.scratch_rows.row(orig_idx as usize);
// Classify against the current K-th-best (the heap top).
@@ -1672,21 +1757,21 @@ impl PartitionedTopKRank {
match classification {
Some(Ordering::Equal) => {
- equal_indices.push(orig_idx);
+ equal_indices.push(sub_idx as u32);
continue;
}
Some(Ordering::Greater) => continue,
Some(Ordering::Less) | None => {
// Heap path: heap not yet full, or row strictly
// better than the current boundary.
- let entry_ref = entry.get_or_insert_with(|| {
- state.heap.register_batch(batch.clone())
+ let entry_ref = heap_entry.get_or_insert_with(|| {
+ state.heap.register_batch(sub_batch.clone())
});
if let Some(EvictedRow {
batch: evicted_batch,
index: evicted_index,
row_bytes: evicted_bytes,
- }) = state.heap.add(entry_ref, row, orig_idx as usize)
+ }) = state.heap.add(entry_ref, row, sub_idx)
{
// Compare the new boundary (post-eviction heap
// top) against the evicted row's bytes — both
@@ -1706,16 +1791,14 @@ impl PartitionedTopKRank {
state.ties.clear();
equal_indices.clear();
} else {
- // Boundary unchanged — evicted row is tied
- // at the (unchanged) boundary; push as a
- // single-row entry.
- let batch_bytes =
- get_record_batch_memory_size(&evicted_batch);
- state.ties.push(TieEntry {
- batch: evicted_batch,
- row_indices: vec![evicted_index as u32],
- batch_bytes,
- });
+ // Boundary unchanged — the evicted row is
+ // still tied at the boundary. Gather just
+ // that row: holding `evicted_batch` would
+ // keep a whole heap batch alive for one row,
+ // and one such entry per input batch would
+ // again make memory grow with the input.
+ let one = UInt32Array::from(vec![evicted_index as u32]);
+ state.push_ties(take_record_batch(&evicted_batch, &one)?);
}
}
replacements += 1;
@@ -1723,21 +1806,34 @@ impl PartitionedTopKRank {
}
}
- if let Some(e) = entry {
+ let registered_with_heap = heap_entry.is_some();
+ if let Some(e) = heap_entry {
state.heap.insert_batch_entry(e);
state.heap.maybe_compact()?;
}
// Commit this batch's ties as a single entry.
if !equal_indices.is_empty() {
- state.ties.push(TieEntry {
- batch: batch.clone(),
- row_indices: equal_indices,
- batch_bytes: input_batch_bytes,
- });
+ // No row of this group reached the heap, so `sub_batch` is
+ // not registered there and reusing it here cannot
+ // double-charge it. Combined with every row having tied,
+ // `sub_batch` already *is* exactly the tie rows — the
+ // gather below would just copy it.
+ let tie_batch = if !registered_with_heap
+ && equal_indices.len() == sub_batch.num_rows()
+ {
+ sub_batch
+ } else {
+ take_record_batch(&sub_batch, &UInt32Array::from(equal_indices))?
+ };
+ state.push_ties(tie_batch);
}
}
+ // Return the drained scratch map (capacity retained) for the next
+ // batch to reuse.
+ self.partition_groups = groups;
+
if replacements > 0 {
self.metrics.row_replacements.add(replacements);
}
@@ -1762,12 +1858,13 @@ impl PartitionedTopKRank {
partition_converter: _,
partition_scratch_rows: _,
mut states,
+ partition_groups: _,
k: _,
batch_size,
} = self;
let _timer = metrics.baseline.elapsed_compute().timer();
- let mut sorted_pks: Vec<OwnedRow> = states.keys().cloned().collect();
+ let mut sorted_pks: Vec<Vec<u8>> = states.keys().cloned().collect();
sorted_pks.sort();
let mut coalescer = BatchCoalescer::new(Arc::clone(&schema), batch_size);
@@ -1779,9 +1876,8 @@ impl PartitionedTopKRank {
coalescer.push_batch(batch)?;
}
for tie in ties {
- let indices = UInt32Array::from(tie.row_indices);
- let tie_batch = take_record_batch(&tie.batch, &indices)?;
- coalescer.push_batch(tie_batch)?;
+ (&tie.batch).record_output(&metrics.baseline);
+ coalescer.push_batch(tie.batch)?;
}
}
coalescer.finish_buffered_batch()?;
@@ -1800,14 +1896,24 @@ impl PartitionedTopKRank {
/// Total memory currently held, including all per-partition states.
fn size(&self) -> usize {
+ // Per partition: the state plus the encoded partition key owned by
+ // the map. The key bytes are a heap allocation the table slot
+ // doesn't cover.
+ let states_contents: usize = self
+ .states
+ .iter()
+ .map(|(pk, state)| pk.capacity() + state.size())
+ .sum();
size_of::<Self>()
+ self.row_converter.size()
+ self.partition_converter.size()
+ self.scratch_rows.size()
+ self.partition_scratch_rows.size()
- + self.states.values().map(|s| s.size()).sum::<usize>()
+ + states_contents
+ self.states.capacity()
- * (size_of::<OwnedRow>() + size_of::<RankPartitionState>())
+ * (size_of::<Vec<u8>>() + size_of::<RankPartitionState>())
+ + self.partition_groups.capacity()
+ * (size_of::<Vec<u8>>() + size_of::<Vec<u32>>())
}
}
@@ -2074,8 +2180,26 @@ impl PartitionedTopKDenseRank {
// value don't re-allocate.
let mut runs = std::mem::take(&mut self.ob_runs);
runs.clear();
+ // Once the partition holds its full K distinct ob values, the
+ // largest of them is the bar a new value must beat, and that bar
+ // only ever gets stricter: with K values held there is no free
+ // slot left for a new (possibly larger) one, and the only way in
+ // from here is to evict that largest and put something strictly
+ // smaller in its place. So a row worse than today's bar is worse
+ // than every later bar too — it is not held now and can never be
+ // admitted — and can be dropped before it costs a bucket. Rows
+ // *equal* to the bar must still go through: that value is one of
+ // the K being held, so its rows belong to a live group.
+ let boundary: Option<&[u8]> = if state.groups.len() == k {
+ state.keys.peek().map(Vec::as_slice)
+ } else {
+ None
+ };
for &orig_idx in &indices {
let ob_row = self.scratch_rows.row(orig_idx as usize);
+ if boundary.is_some_and(|b| ob_row.as_ref() > b) {
+ continue;
+ }
runs.entry_ref(ob_row.as_ref()).or_default().push(orig_idx);
}
@@ -2262,7 +2386,9 @@ mod tests {
use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow_schema::SortOptions;
- use datafusion_common::assert_batches_eq;
+ use datafusion_common::{assert_batches_eq, exec_datafusion_err};
+ use datafusion_execution::memory_pool::GreedyMemoryPool;
+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_physical_expr::{DynamicFilterTracking, expressions::col};
use futures::TryStreamExt;
@@ -2967,10 +3093,7 @@ mod tests {
val_sort_options: SortOptions,
val_nullable: bool,
) -> Result<(Arc<Schema>, PartitionedTopK)> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("pk", DataType::Int32, false),
- Field::new("val", DataType::Int32, val_nullable),
- ]));
+ let schema = pk_val_schema(val_nullable);
let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
let pk_sort_expr = PhysicalSortExpr {
@@ -2999,6 +3122,358 @@ mod tests {
Ok((schema, state))
}
+ /// `PartitionedTopKRank` must not charge (or pin) a whole input batch
+ /// once per partition key that batch touches.
+ ///
+ /// Regression: the heap used to be handed `batch.clone()` — the full
+ /// input batch — rather than a gather of just that partition's rows,
+ /// so a batch spanning P partitions was counted P times over. With 500
+ /// partitions in one batch that reported ~516x the batch size, against
+ /// ~15x for the equivalent `ROW_NUMBER` state on identical input.
+ #[tokio::test]
+ async fn test_partitioned_topk_rank_size_is_not_per_partition_batch() -> Result<()> {
+ // The over-count factor was exactly the number of partitions sharing
+ // a batch, so P is what makes the bug visible at all. 500 turns the
+ // rank/row_number size ratio from ~1x into ~35x — far outside the
+ // range any accounting change could drift into.
+ const P: i32 = 500;
+ // 8 rows per partition: enough for every partition's k=2 heap to fill
+ // and then evict, so the heaps hold real state rather than sitting
+ // half-empty in the fill phase.
+ const ROWS: i32 = 4000;
+
+ // Distinct values throughout: no ties, so RANK retains exactly what
+ // ROW_NUMBER does and the two sizes are directly comparable. k=2 is
+ // the smallest k with a distinct fill phase before eviction begins.
+ let pks: Vec<i32> = (0..ROWS).map(|i| i % P).collect();
+ let vals: Vec<i32> = (0..ROWS).map(|i| i / P).collect();
+
+ let (schema, mut rn) = build_partitioned_topk(2)?;
+ rn.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;
+ let rn_size = rn.size();
+
+ let (schema, mut rk) = build_partitioned_topk_rank(2)?;
+ rk.insert_batch(&pk_val_batch(&schema, pks, vals)?)?;
+ let rk_size = rk.size();
+
+ // Measured ratio here is ~1.04 (18.1x vs 17.4x of the input batch);
+ // the bug produced ~35x. A 2x bar sits more than an order of magnitude
+ // clear of both, so this neither flakes when the size calculation is
+ // legitimately adjusted nor misses a reintroduced per-partition factor.
+ assert!(
+ rk_size <= rn_size * 2,
+ "RANK reported {rk_size} bytes for the same retained rows that \
+ ROW_NUMBER reported {rn_size} bytes for; a per-partition factor \
+ has crept back in"
+ );
+ Ok(())
+ }
+
+ /// A boundary tie must retain only the tied rows, not the batch they
+ /// arrived in.
+ ///
+ /// Regression: `TieEntry` used to hold the source `RecordBatch` plus
+ /// indices, so one tied row pinned — and was charged for — a whole
+ /// input batch, for as long as the boundary held. Across a stream of
+ /// batches that made retained memory grow with the *input* size rather
+ /// than with `K + ties`.
+ #[tokio::test]
+ async fn test_partitioned_topk_rank_ties_do_not_pin_input_batches() -> Result<()> {
+ // k=1 makes every retained row after the very first one a boundary
+ // tie, which is the state under test.
+ let (schema, mut rk) = build_partitioned_topk_rank(1)?;
+
+ // Each batch carries exactly one row tied at the boundary and 999
+ // rows that are strictly worse and must be dropped. The 1000:1 ratio
+ // is the point: holding the tied row costs a few bytes, holding the
+ // batch it arrived in costs ~8 KB, so the two outcomes cannot be
+ // confused.
+ let mut vals = vec![100; 1000];
+ vals[0] = 7;
+ let batch = pk_val_batch(&schema, vec![1; 1000], vals)?;
+
+ rk.insert_batch(&batch)?;
+ let after_first = rk.size();
+ for _ in 0..7 {
+ rk.insert_batch(&batch)?;
+ }
+ let after_eight = rk.size();
+
+ // Seven more batches retain seven more single rows. Correct growth is
+ // ~8 bytes per batch (~56 total, just the tie-list slots); the bug grew
+ // by a whole ~8 KB batch each time (~56 KB total). A 2000-byte bar sits
+ // between the two with more than an order of magnitude of clearance on
+ // each side.
+ let growth = after_eight - after_first;
+ assert!(
+ growth < 2000,
+ "tie list grew {growth} bytes over 7 batches that contributed \
+ 1 row each; it is holding the source batches"
+ );
+ Ok(())
+ }
+
+ /// The reservation must clear a *bounded* memory pool, not merely
+ /// report a plausible `size()`.
+ ///
+ /// The two tests above check what `size()` computes; this one checks
+ /// how that number is actually spent — `try_resize` against a real
+ /// pool, which is what returned `ResourcesExhausted` to the user.
+ ///
+ /// No byte constant is asserted. The limit is derived from the input
+ /// at run time, because the property under test is a ratio: retention
+ /// must be a small multiple of *one* input batch no matter how many
+ /// batches stream through, where whole-batch retention was
+ /// (partitions x batch) and needed ~P times as much. Only the ratio
+ /// has to hold as the size accounting is legitimately adjusted.
+ ///
+ /// Scope: the insert path only. `emit` drops the reservation before
+ /// it materializes ties, so emit-time growth is out of reach here.
+ #[tokio::test]
+ async fn test_partitioned_topk_rank_runs_under_bounded_memory_pool() -> Result<()> {
+ // P is the whole point: the old code pinned and charged one copy of
+ // each batch per partition that batch touched, so it needed ~P x the
+ // batch where the fix needs a constant multiple of it.
+ const P: i32 = 256;
+ const ROWS_PER_PARTITION: i32 = 32;
+ const ROWS: i32 = P * ROWS_PER_PARTITION;
+ const BATCHES: i32 = 16;
+ const K: usize = 2;
+
+ let schema = pk_val_schema(false);
+ let pks: Vec<i32> = (0..ROWS).map(|i| i % P).collect();
+ // Batch b's values all beat batch b-1's, so every batch is admitted
+ // by every partition and the batch it displaces must be released —
+ // the case that separates "charged once per batch" from "charged
+ // once per partition per batch". Within a batch a partition's
+ // values are distinct, so nothing ties at the boundary and each
+ // partition keeps exactly K rows throughout.
+ let vals = |b: i32| -> Vec<i32> {
+ (0..ROWS)
+ .map(|i| (BATCHES - b) * ROWS_PER_PARTITION + i / P)
+ .collect()
+ };
+
+ // A budget of 32 batches, derived from the data so no byte constant
+ // here goes stale. The operator legitimately holds ~8x the batch
+ // (row-encoding scratch for one batch, plus per-partition heap and
+ // map overhead), so this clears it ~4x over; the old whole-batch
+ // retention needed ~P = 256x the batch, which is 8x past the limit.
+ let batch_bytes =
+ get_record_batch_memory_size(&pk_val_batch(&schema, pks.clone(), vals(0))?);
+ let limit = 32 * batch_bytes;
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(limit)))
+ .build_arc()?;
+
+ let (schema, mut rk) = build_partitioned_topk_rank_with_runtime(K, &runtime)?;
+
+ let mut after_first = 0;
+ for b in 0..BATCHES {
+ let batch = pk_val_batch(&schema, pks.clone(), vals(b))?;
+ rk.insert_batch(&batch).map_err(|e| {
+ exec_datafusion_err!(
+ "batch {b} of {BATCHES} failed under a {limit}-byte pool: {e}"
+ )
+ })?;
+ if b == 0 {
+ after_first = rk.size();
+ }
+ }
+
+ // Flat, not merely under the limit: the last 15 batches must not
+ // have added retention (in practice this is exactly equal). The 2x
+ // slack absorbs map and tie-list capacity growth, but not a batch's
+ // worth of pinned rows.
+ assert!(
+ rk.size() <= after_first * 2,
+ "retention grew from {after_first} to {} bytes across {BATCHES} \
+ batches; it tracks the input rather than a bounded window of it",
+ rk.size()
+ );
+
+ // A pool that was never pressed proves nothing, so pin down what the
+ // operator produced: K rows per partition, drawn from the last batch,
+ // whose values are the smallest seen.
+ let expected: Vec<(i32, i32)> = (0..P)
+ .flat_map(|pk| [(pk, ROWS_PER_PARTITION), (pk, ROWS_PER_PARTITION + 1)])
+ .collect();
+ assert_eq!(sorted_pk_val(rk.emit()?).await?, expected);
+ Ok(())
+ }
+
+ /// One deterministic pseudo-random input shape for the differential
+ /// tests below: a `k`, and the `(pks, vals)` batches to feed in.
+ ///
+ /// Shapes are deliberately tiny. These bugs live in how ties, partitions
+ /// and batch boundaries interleave, not in volume, so many small shapes
+ /// cover far more of that space than a few large ones — and they keep the
+ /// O(n^2) brute-force reference cheap.
+ struct DiffShape {
+ seed: u64,
+ k: usize,
+ n_partitions: i32,
+ n_values: i32,
+ batches: Vec<(Vec<i32>, Vec<i32>)>,
+ }
+
+ impl std::fmt::Display for DiffShape {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "seed={} k={} partitions={} values={} batches={}",
+ self.seed,
+ self.k,
+ self.n_partitions,
+ self.n_values,
+ self.batches.len()
+ )
+ }
+ }
+
+ impl DiffShape {
+ /// `max_values` caps the ORDER BY value domain. Keep it near `k` so
+ /// rows tie above, at, and below the boundary frequently rather than
+ /// by chance; raise it for an operator that bounds *distinct* values
+ /// and so needs more than `k` of them before it will evict anything.
+ fn new(seed: u64, max_values: i32) -> Self {
+ use rand::rngs::StdRng;
+ use rand::{Rng, SeedableRng};
+
+ let mut rng = StdRng::seed_from_u64(seed);
+ let k = rng.random_range(1..5usize);
+ let n_partitions = rng.random_range(1..4i32);
+ let n_values = rng.random_range(1..max_values);
+ let n_batches = rng.random_range(1..5usize);
+
+ let batches = (0..n_batches)
+ .map(|_| {
+ let rows = rng.random_range(1..12usize);
+ let pks = (0..rows)
+ .map(|_| rng.random_range(0..n_partitions))
+ .collect();
+ let vals = (0..rows).map(|_| rng.random_range(0..n_values)).collect();
+ (pks, vals)
+ })
+ .collect();
+
+ Self {
+ seed,
+ k,
+ n_partitions,
+ n_values,
+ batches,
+ }
+ }
+
+ /// Every `(pk, val)` fed in, across all batches.
+ fn all_rows(&self) -> Vec<(i32, i32)> {
+ self.batches
+ .iter()
+ .flat_map(|(pks, vals)| pks.iter().copied().zip(vals.iter().copied()))
+ .collect()
+ }
+
+ /// Brute-force reference: the sorted rows a `<= k` filter keeps, given
+ /// `rank_of(all_rows, pk, val)` for the ranking function under test.
+ fn expected(
+ &self,
+ rank_of: impl Fn(&[(i32, i32)], i32, i32) -> usize,
+ ) -> Vec<(i32, i32)> {
+ let all = self.all_rows();
+ let mut kept: Vec<(i32, i32)> = all
+ .iter()
+ .copied()
+ .filter(|&(pk, val)| rank_of(&all, pk, val) <= self.k)
+ .collect();
+ kept.sort_unstable();
+ kept
+ }
+ }
+
+ /// Drain an operator's output into sorted `(pk, val)` pairs, ready to
+ /// compare against [`DiffShape::expected`].
+ async fn sorted_pk_val(stream: SendableRecordBatchStream) -> Result<Vec<(i32, i32)>> {
+ let batches: Vec<RecordBatch> = stream.try_collect().await?;
+ let mut rows: Vec<(i32, i32)> = Vec::new();
+ for b in &batches {
+ let pk = b.column(0).as_primitive::<arrow::datatypes::Int32Type>();
+ let val = b.column(1).as_primitive::<arrow::datatypes::Int32Type>();
+ for i in 0..b.num_rows() {
+ rows.push((pk.value(i), val.value(i)));
+ }
+ }
+ rows.sort_unstable();
+ Ok(rows)
+ }
+
+ /// Randomized differential test for `PartitionedTopKRank`.
+ ///
+ /// The retention rule is subtle — a K-bounded heap plus a boundary-tie
+ /// list that must be discarded the moment the K-th-best ORDER BY value
+ /// improves — and it turns on how partitions, ties and batch boundaries
+ /// interleave. 64 seeds runs in ~10 ms; deleting the tie-clear on a
+ /// boundary shift is caught by seed 0.
+ #[tokio::test]
+ async fn test_partitioned_topk_rank_matches_bruteforce() -> Result<()> {
+ for seed in 0..64u64 {
+ let shape = DiffShape::new(seed, 6);
+ let (schema, mut state) = build_partitioned_topk_rank(shape.k)?;
+ for (pks, vals) in &shape.batches {
+ state.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;
+ }
+
+ // RANK: 1 + the number of strictly smaller rows.
+ let expected = shape.expected(|rows, pk, val| {
+ 1 + rows.iter().filter(|&&(p, v)| p == pk && v < val).count()
+ });
+
+ assert_eq!(sorted_pk_val(state.emit()?).await?, expected, "{shape}");
+ }
+ Ok(())
+ }
+
+ /// Randomized differential test for `PartitionedTopKDenseRank`.
+ ///
+ /// Same harness as [`test_partitioned_topk_rank_matches_bruteforce`],
+ /// differing only in the ranking formula and a wider value domain:
+ /// DENSE_RANK bounds *distinct* values, so a partition needs more than
+ /// `k` of them before it evicts anything, and eviction is what the
+ /// admission pre-filter guards. A pre-filter that wrongly rejects
+ /// boundary-equal rows is caught by seed 6.
+ #[tokio::test]
+ async fn test_partitioned_topk_dense_rank_matches_bruteforce() -> Result<()> {
+ for seed in 0..64u64 {
+ let shape = DiffShape::new(seed, 8);
+ let (schema, mut state) = build_partitioned_topk_dense_rank(shape.k)?;
+ for (pks, vals) in &shape.batches {
+ state.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;
+ }
+
+ // DENSE_RANK: 1 + the number of *distinct* strictly smaller values.
+ let expected = shape.expected(|rows, pk, val| {
+ 1 + rows
+ .iter()
+ .filter(|&&(p, v)| p == pk && v < val)
+ .map(|&(_, v)| v)
+ .collect::<std::collections::BTreeSet<_>>()
+ .len()
+ });
+
+ assert_eq!(sorted_pk_val(state.emit()?).await?, expected, "{shape}");
+ }
+ Ok(())
+ }
+
+ /// The `(pk Int32, val Int32)` schema every `PartitionedTopK*` test
+ /// builds against. `val_nullable` is what the null-ordering tests vary.
+ fn pk_val_schema(val_nullable: bool) -> Arc<Schema> {
+ Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::Int32, false),
+ Field::new("val", DataType::Int32, val_nullable),
+ ]))
+ }
+
fn pk_val_batch(
schema: &Arc<Schema>,
pks: Vec<i32>,
@@ -3341,10 +3816,31 @@ mod tests {
val_sort_options: SortOptions,
val_nullable: bool,
) -> Result<(Arc<Schema>, PartitionedTopKRank)> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("pk", DataType::Int32, false),
- Field::new("val", DataType::Int32, val_nullable),
- ]));
+ build_partitioned_topk_rank_inner(
+ k,
+ val_sort_options,
+ val_nullable,
+ &Arc::new(RuntimeEnv::default()),
+ )
+ }
+
+ /// Variant of [`build_partitioned_topk_rank`] that takes the
+ /// [`RuntimeEnv`] to register the reservation against, so a test can
+ /// bound the memory pool the operator draws from.
+ fn build_partitioned_topk_rank_with_runtime(
+ k: usize,
+ runtime: &Arc<RuntimeEnv>,
+ ) -> Result<(Arc<Schema>, PartitionedTopKRank)> {
+ build_partitioned_topk_rank_inner(k, SortOptions::default(), false, runtime)
+ }
+
+ fn build_partitioned_topk_rank_inner(
+ k: usize,
+ val_sort_options: SortOptions,
+ val_nullable: bool,
+ runtime: &Arc<RuntimeEnv>,
+ ) -> Result<(Arc<Schema>, PartitionedTopKRank)> {
+ let schema = pk_val_schema(val_nullable);
let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
let pk_sort_expr = PhysicalSortExpr {
@@ -3367,7 +3863,7 @@ mod tests {
order_expr,
k,
8, // batch_size
- &Arc::new(RuntimeEnv::default()),
+ runtime,
&ExecutionPlanMetricsSet::new(),
)?;
Ok((schema, state))
@@ -3733,10 +4229,7 @@ mod tests {
val_sort_options: SortOptions,
val_nullable: bool,
) -> Result<(Arc<Schema>, PartitionedTopKDenseRank)> {
- let schema = Arc::new(Schema::new(vec![
- Field::new("pk", DataType::Int32, false),
- Field::new("val", DataType::Int32, val_nullable),
- ]));
+ let schema = pk_val_schema(val_nullable);
let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
let pk_sort_expr = PhysicalSortExpr {