Skip to content

Apache DataFusion SQL Query Engine

perf: interleave retained rows at emit instead of a take per group entry (#25468)

Subham Singhalcee15b7cca30
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index cec3681de9..b281e49873 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -1951,29 +1951,75 @@ struct DenseRankPartitionState {
/// INVARIANT: `keys` and `groups.keys()` hold the same set. Every
/// insertion into / removal from `groups` must mirror into `keys`.
keys: BinaryHeap<Vec<u8>>,
+ /// Running total of the heap allocations owned by the *contents* of
+ /// `groups` and `keys`: the key bytes, the per-key `Vec<GroupEntry>`
+ /// buffers, and each entry's `row_indices`. Excludes the two
+ /// containers' own tables, which `capacity()` reports in O(1).
+ ///
+ /// INVARIANT: equals `recompute_contents_bytes` (test-only, so not
+ /// linkable from rustdoc). Every mutation of `groups` or `keys` must
+ /// adjust it; the
+ /// `test_partitioned_topk_dense_rank_contents_bytes_tracks_recompute`
+ /// test checks this against a full recompute after a randomized
+ /// workload.
+ contents_bytes: usize,
}
impl DenseRankPartitionState {
fn size(&self) -> usize {
let table_overhead = self.groups.capacity()
* (size_of::<Vec<u8>>() + size_of::<Vec<GroupEntry>>());
- let contents: usize = self
+ // The heap's backing Vec: one `Vec<u8>` slot per reserved element.
+ let keys_overhead = self.keys.capacity() * size_of::<Vec<u8>>();
+ table_overhead + self.contents_bytes + keys_overhead
+ }
+
+ /// Bytes owned by one entry's `row_indices` buffer.
+ fn entry_bytes(entry: &GroupEntry) -> usize {
+ entry.row_indices.capacity() * size_of::<u32>()
+ }
+
+ /// Track a previously unseen ob value, holding `run_indices` as its
+ /// first (and so far only) entry, and charge everything it allocates.
+ ///
+ /// `keys` and `groups` each own a copy of the key bytes; the clone's
+ /// capacity is read after the fact rather than assumed equal to the
+ /// original's, so the charge matches what was really allocated.
+ fn insert_new_group(
+ &mut self,
+ ob_key: Vec<u8>,
+ run_indices: Vec<u32>,
+ batch_id: u32,
+ ) {
+ let key_copy = ob_key.clone();
+ self.contents_bytes += key_copy.capacity() + ob_key.capacity();
+ self.keys.push(key_copy);
+
+ let entry = GroupEntry {
+ row_indices: run_indices,
+ batch_id,
+ };
+ self.contents_bytes += Self::entry_bytes(&entry);
+ let entries = vec![entry];
+ self.contents_bytes += entries.capacity() * size_of::<GroupEntry>();
+ self.groups.insert(ob_key, entries);
+ }
+
+ /// The value [`Self::contents_bytes`] must hold, computed the slow
+ /// way. Used to assert the incremental accounting in tests.
+ #[cfg(test)]
+ fn recompute_contents_bytes(&self) -> usize {
+ let groups: usize = self
.groups
.iter()
.map(|(key, entries)| {
key.capacity()
+ entries.capacity() * size_of::<GroupEntry>()
- + entries
- .iter()
- .map(|e| e.row_indices.capacity() * size_of::<u32>())
- .sum::<usize>()
+ + entries.iter().map(Self::entry_bytes).sum::<usize>()
})
.sum();
- // `keys` duplicates every key's bytes; charge for them plus the
- // heap's backing Vec (one `Vec<u8>` slot per reserved element).
- let keys_overhead: usize = self.keys.capacity() * size_of::<Vec<u8>>()
- + self.keys.iter().map(|k| k.capacity()).sum::<usize>();
- table_overhead + contents + keys_overhead
+ // `keys` duplicates every key's bytes.
+ groups + self.keys.iter().map(|k| k.capacity()).sum::<usize>()
}
}
@@ -2207,24 +2253,23 @@ impl PartitionedTopKDenseRank {
// new `GroupEntry` (one entry per contributing batch).
if let Some(entries) = state.groups.get_mut(&ob_key) {
batch_entry.uses += 1;
- entries.push(GroupEntry {
+ let before = entries.capacity() * size_of::<GroupEntry>();
+ let entry = GroupEntry {
row_indices: run_indices,
batch_id,
- });
+ };
+ state.contents_bytes += DenseRankPartitionState::entry_bytes(&entry);
+ entries.push(entry);
+ // The push may have grown the buffer.
+ state.contents_bytes +=
+ entries.capacity() * size_of::<GroupEntry>() - before;
continue;
}
// Case B: new ob, room available.
if state.groups.len() < k {
batch_entry.uses += 1;
- state.keys.push(ob_key.clone());
- state.groups.insert(
- ob_key,
- vec![GroupEntry {
- row_indices: run_indices,
- batch_id,
- }],
- );
+ state.insert_new_group(ob_key, run_indices, batch_id);
continue;
}
@@ -2233,12 +2278,23 @@ impl PartitionedTopKDenseRank {
let max_key = state.keys.peek().expect("state.groups has k >= 1 keys");
if ob_key.as_slice() < max_key.as_slice() {
// Evict the entire max-key group, from both the map
- // and its ordered mirror.
+ // and its ordered mirror. `keys` and `groups` own
+ // separate copies of the key bytes, so both are
+ // uncharged — with their own capacities, which is why
+ // `remove_entry` is used to recover the map's copy
+ // rather than assuming it matches the popped one.
let evicted_key = state.keys.pop().expect("max key present");
- let evicted = state
+ let (map_key, evicted) = state
.groups
- .remove(&evicted_key)
+ .remove_entry(&evicted_key)
.expect("keys mirrors groups");
+ state.contents_bytes -= evicted_key.capacity()
+ + map_key.capacity()
+ + evicted.capacity() * size_of::<GroupEntry>()
+ + evicted
+ .iter()
+ .map(DenseRankPartitionState::entry_bytes)
+ .sum::<usize>();
for e in &evicted {
replacements += e.row_indices.len();
if e.batch_id == batch_id {
@@ -2252,14 +2308,7 @@ impl PartitionedTopKDenseRank {
}
}
batch_entry.uses += 1;
- state.keys.push(ob_key.clone());
- state.groups.insert(
- ob_key,
- vec![GroupEntry {
- row_indices: run_indices,
- batch_id,
- }],
- );
+ state.insert_new_group(ob_key, run_indices, batch_id);
}
// else: ob >= max — drop the whole run.
}
@@ -2311,11 +2360,41 @@ impl PartitionedTopKDenseRank {
let mut sorted_pks: Vec<Vec<u8>> = states.keys().cloned().collect();
sorted_pks.sort();
- let mut coalescer = BatchCoalescer::new(Arc::clone(&schema), batch_size);
+ let mut out: Vec<Result<RecordBatch>> = Vec::new();
+ // Gather every retained row with a single `interleave_record_batch`
+ // per output batch rather than one `take_record_batch` per
+ // `GroupEntry`. A group entry holds only the rows one source batch
+ // contributed at one ob value, so entries are numerous and tiny —
+ // with P partitions, K distinct ob values and B contributing
+ // batches there are up to P × K × B of them, and gathering each
+ // one separately builds and tears down that many `RecordBatch`es.
+ // `interleave` takes `(batch_pos, row)` pairs across *different*
+ // source batches in one call, which is exactly the shape here.
+ //
+ // The pairs are pushed in emit order — partitions in sorted key
+ // order, ob values ascending within a partition, entries in
+ // insertion order within an ob value — so the interleaved output
+ // is already ordered and needs no post-sort.
+ //
+ // `batch_refs` holds only the batches the current chunk references
+ // and is rebuilt per chunk: for Dictionary columns `interleave`
+ // does work for every input array it is handed, referenced or not,
+ // so passing the whole store to each chunk made emit
+ // O(chunks × store batches). A batch's slot is resolved once per
+ // entry, not per row.
+ let mut batch_refs: Vec<&RecordBatch> = Vec::new();
+ let mut batch_id_pos: HashMap<u32, usize> = HashMap::new();
+ let mut indices: Vec<(usize, usize)> = Vec::with_capacity(batch_size);
+
+ // Chunk at `batch_size` so the operator emits the same batch sizes
+ // as before and no single `interleave` output exceeds `batch_size`.
for pk in sorted_pks {
- let DenseRankPartitionState { groups, keys: _ } =
- states.remove(&pk).expect("key from states.keys()");
+ let DenseRankPartitionState {
+ groups,
+ keys: _,
+ contents_bytes: _,
+ } = states.remove(&pk).expect("key from states.keys()");
// Sort the <= K distinct ob keys so rows emit ascending
// (byte-comparable encoding == sort order).
let mut sorted_obs: Vec<(Vec<u8>, Vec<GroupEntry>)> =
@@ -2327,16 +2406,32 @@ impl PartitionedTopKDenseRank {
.get(entry.batch_id)
.expect("retained batch_id present in store")
.batch;
- let indices = UInt32Array::from(entry.row_indices);
- let sub = take_record_batch(batch, &indices)?;
- coalescer.push_batch(sub)?;
+ // Resolved lazily so a chunk boundary inside the entry
+ // re-registers its batch in the next chunk's slice.
+ let mut array_pos = None;
+ for row in entry.row_indices {
+ let pos = *array_pos.get_or_insert_with(|| {
+ *batch_id_pos.entry(entry.batch_id).or_insert_with(|| {
+ batch_refs.push(batch);
+ batch_refs.len() - 1
+ })
+ });
+ indices.push((pos, row as usize));
+ if indices.len() == batch_size {
+ let b = interleave_record_batch(&batch_refs, &indices)?;
+ (&b).record_output(&metrics.baseline);
+ out.push(Ok(b));
+ indices.clear();
+ batch_refs.clear();
+ batch_id_pos.clear();
+ array_pos = None;
+ }
+ }
}
}
}
- coalescer.finish_buffered_batch()?;
-
- let mut out: Vec<Result<RecordBatch>> = Vec::new();
- while let Some(b) = coalescer.next_completed_batch() {
+ if !indices.is_empty() {
+ let b = interleave_record_batch(&batch_refs, &indices)?;
(&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -2382,8 +2477,11 @@ impl PartitionedTopKDenseRank {
mod tests {
use super::*;
use crate::metrics::MetricValue;
- use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};
- use arrow::datatypes::{DataType, Field, Schema};
+ use arrow::array::{
+ AsArray, BooleanArray, Float64Array, Int32Array, StringArray,
+ StringDictionaryBuilder,
+ };
+ use arrow::datatypes::{DataType, Field, Int32Type, Schema};
use arrow_schema::SortOptions;
use datafusion_common::{assert_batches_eq, exec_datafusion_err};
use datafusion_execution::memory_pool::GreedyMemoryPool;
@@ -3392,16 +3490,26 @@ mod tests {
/// Drain an operator's output into sorted `(pk, val)` pairs, ready to
/// compare against [`DiffShape::expected`].
+ ///
+ /// Asserts the rows arrived already in `(pk ASC, val ASC)` order before
+ /// sorting them. That order is load-bearing rather than cosmetic — it is
+ /// what `PartitionedTopKExec::compute_properties` advertises, and so what
+ /// lets the window above it run `mode=Sorted` with no `SortExec` — and
+ /// the content comparison this feeds would otherwise sort it away.
async fn sorted_pk_val(stream: SendableRecordBatchStream) -> Result<Vec<(i32, i32)>> {
let batches: Vec<RecordBatch> = stream.try_collect().await?;
let mut rows: Vec<(i32, i32)> = Vec::new();
for b in &batches {
- let pk = b.column(0).as_primitive::<arrow::datatypes::Int32Type>();
- let val = b.column(1).as_primitive::<arrow::datatypes::Int32Type>();
+ let pk = b.column(0).as_primitive::<Int32Type>();
+ let val = b.column(1).as_primitive::<Int32Type>();
for i in 0..b.num_rows() {
rows.push((pk.value(i), val.value(i)));
}
}
+ assert!(
+ rows.windows(2).all(|w| w[0] <= w[1]),
+ "emitted rows are not in (pk, val) order: {rows:?}"
+ );
rows.sort_unstable();
Ok(rows)
}
@@ -3464,6 +3572,42 @@ mod tests {
Ok(())
}
+ /// `DenseRankPartitionState::contents_bytes` is maintained
+ /// incrementally at four mutation sites (append to an existing ob
+ /// group, insert a new one with room, evict-then-insert, and the
+ /// key/entry buffer growth each can trigger). Drift there would
+ /// silently give the reservation a wrong total, so check it against a
+ /// recompute over the same randomized workload the correctness
+ /// differential test uses — its shapes are tuned to exercise
+ /// eviction, which is the case with the most bookkeeping.
+ #[tokio::test]
+ async fn test_partitioned_topk_dense_rank_contents_bytes_tracks_recompute()
+ -> Result<()> {
+ let mut saw_eviction = false;
+ for seed in 0..64u64 {
+ let shape = DiffShape::new(seed, 8);
+ let (schema, mut state) = build_partitioned_topk_dense_rank(shape.k)?;
+ for (pks, vals) in &shape.batches {
+ state.insert_batch(&pk_val_batch(&schema, pks.clone(), vals.clone())?)?;
+ // Check after every batch, not just at the end: a
+ // compensating pair of errors within one batch would
+ // survive an end-only assertion.
+ for (pk, partition) in &state.states {
+ assert_eq!(
+ partition.contents_bytes,
+ partition.recompute_contents_bytes(),
+ "seed {seed}, partition {pk:?}: {shape}"
+ );
+ }
+ }
+ saw_eviction |= state.metrics.row_replacements.value() > 0;
+ }
+ // Guards the guard: if the shapes stopped evicting, case C would
+ // go unchecked and this test would still pass.
+ assert!(saw_eviction, "workload never evicted a group");
+ Ok(())
+ }
+
/// The `(pk Int32, val Int32)` schema every `PartitionedTopK*` test
/// builds against. `val_nullable` is what the null-ordering tests vary.
fn pk_val_schema(val_nullable: bool) -> Arc<Schema> {
@@ -4453,6 +4597,97 @@ mod tests {
Ok(())
}
+ /// `emit` rebuilds the `interleave` batch slice per chunk, so a
+ /// Dictionary column must survive chunk boundaries that change which
+ /// source batches are in the slice — including a boundary that falls
+ /// *inside* one group entry (the four `v06` rows), whose batch must be
+ /// re-registered in the next chunk's slice. `batch_size` is 8, so the
+ /// 20 retained rows emit as 8 + 8 + 4, with the second chunk starting
+ /// mid-entry and mixing both source batches.
+ #[tokio::test]
+ async fn test_partitioned_topk_dense_rank_emit_dictionary_spans_chunks() -> Result<()>
+ {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::Int32, false),
+ Field::new(
+ "val",
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+ false,
+ ),
+ ]));
+
+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
+ let partition_sort_fields = build_sort_fields(
+ &[PhysicalSortExpr {
+ expr: Arc::clone(&pk_expr),
+ options: SortOptions::default(),
+ }],
+ &schema,
+ )?;
+ let order_expr = LexOrdering::from([PhysicalSortExpr {
+ expr: col("val", schema.as_ref())?,
+ options: SortOptions::default(),
+ }]);
+
+ let mut state = PartitionedTopKDenseRank::try_new(
+ 0,
+ Arc::clone(&schema),
+ vec![pk_expr],
+ partition_sort_fields,
+ order_expr,
+ 20, // k: large enough to retain every distinct value below
+ 8, // batch_size
+ &Arc::new(RuntimeEnv::default()),
+ &ExecutionPlanMetricsSet::new(),
+ )?;
+
+ let dict_batch = |vals: &[&str]| -> Result<RecordBatch> {
+ let mut builder = StringDictionaryBuilder::<Int32Type>::new();
+ for v in vals {
+ builder.append_value(v);
+ }
+ Ok(RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1; vals.len()])),
+ Arc::new(builder.finish()),
+ ],
+ )?)
+ };
+
+ // Two source batches (two distinct `batch_id`s), all under a single
+ // partition key; `v06` is one entry of four rows at batch 1.
+ state.insert_batch(&dict_batch(&[
+ "v00", "v01", "v02", "v03", "v04", "v05", "v06", "v06", "v06", "v06",
+ ])?)?;
+ state.insert_batch(&dict_batch(&[
+ "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
+ ])?)?;
+
+ let results: Vec<_> = state.emit()?.try_collect().await?;
+ assert_eq!(
+ results.iter().map(|b| b.num_rows()).collect::<Vec<_>>(),
+ vec![8, 8, 4]
+ );
+ let mut emitted = Vec::new();
+ for b in &results {
+ let val_col = b.column(1).as_dictionary::<Int32Type>();
+ let dict_values = val_col.values().as_string::<i32>();
+ for key in val_col.keys().values() {
+ emitted.push(dict_values.value(*key as usize).to_string());
+ }
+ }
+ let expected: Vec<String> = [
+ "v00", "v01", "v02", "v03", "v04", "v05", "v06", "v06", "v06", "v06", "v10",
+ "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
+ ]
+ .iter()
+ .map(|s| s.to_string())
+ .collect();
+ assert_eq!(emitted, expected, "rows must emit in ob-sorted order");
+ Ok(())
+ }
+
/// DENSE_RANK-specific: eviction removes the entire max group when
/// a strictly-smaller distinct ob arrives. Multiple rows at the
/// evicted key all disappear.