Skip to content

Apache DataFusion SQL Query Engine

fix: leave memory for others when a spilled sort merges its runs (#26264)

liuxiaoyuadffd19e2c6c
diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge.rs b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
index ebdecb90f3..041e3d064d 100644
--- a/datafusion/physical-plan/src/sorts/multi_level_merge.rs
+++ b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
@@ -15,7 +15,7 @@ use std::task::{Context, Poll};
use arrow::datatypes::SchemaRef;
use datafusion_common::{Result, internal_err, resources_err};
-use datafusion_execution::memory_pool::{MemoryReservation, MergeMemoryPool};
+use datafusion_execution::memory_pool::{MemoryPool, MemoryReservation, MergeMemoryPool};
use crate::sorts::builder::try_grow_reservation_to_at_least;
use crate::sorts::sort::get_reserved_bytes_for_record_batch_size;
@@ -28,6 +28,24 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;
use futures::TryStreamExt;
use futures::{Stream, StreamExt};
+/// Leave memory for others sharing the execution pool when a merge seats
+/// more than the minimum two runs.
+#[derive(Debug, Clone)]
+pub(crate) struct FanInHeadroom {
+ /// The execution pool: the parent of the merge pool when there is one.
+ pub(crate) pool: Arc<dyn MemoryPool>,
+ /// Other partitions share `pool`, even if they hold nothing right now.
+ pub(crate) siblings: bool,
+}
+
+impl FanInHeadroom {
+ /// Whether another consumer may need the pool: sibling partitions exist,
+ /// or the pool holds more than the `owned` bytes this merge holds in it.
+ fn is_contended(&self, owned: usize) -> bool {
+ self.siblings || self.pool.reserved() > owned
+ }
+}
+
/// Merges a stream of sorted cursors and record batches into a single sorted stream
///
/// This is a wrapper around [`SortPreservingMergeStream`](crate::sorts::merge::SortPreservingMergeStream)
@@ -148,6 +166,11 @@ pub(crate) struct MultiLevelMergeBuilder {
merge_pool: Option<Arc<MergeMemoryPool>>,
/// Leave memory for the aggregate consuming the merged spill rows.
reserve_replay_headroom: bool,
+ /// When set and the pool is shared, seat runs beyond the minimum two only
+ /// if the pool can admit their buffers plus as much again as they exceed
+ /// the retained workspace, so a merge does not take the memory that other
+ /// partitions sharing the pool need.
+ fan_in_headroom: Option<FanInHeadroom>,
/// Avoid merging more runs than needed before the final replay pass.
size_intermediate_merges: bool,
fetch: Option<usize>,
@@ -191,6 +214,7 @@ impl MultiLevelMergeBuilder {
reservation,
merge_pool: None,
reserve_replay_headroom: false,
+ fan_in_headroom: None,
size_intermediate_merges: false,
enable_round_robin_tie_breaker,
fetch,
@@ -216,6 +240,17 @@ impl MultiLevelMergeBuilder {
self
}
+ /// Leave headroom when seating runs beyond the minimum two while the pool
+ /// is shared. The minimum merge is admitted against the pool alone, as
+ /// without headroom.
+ pub(super) fn with_fan_in_headroom(
+ mut self,
+ headroom: Option<FanInHeadroom>,
+ ) -> Self {
+ self.fan_in_headroom = headroom;
+ self
+ }
+
pub(super) fn with_intermediate_merge_sizing(
mut self,
min_spill_batch_rows: Option<usize>,
@@ -584,6 +619,20 @@ impl MultiLevelMergeBuilder {
// allocation, preventing starvation under memory pressure.
let mut total_needed: usize = 0;
let mut accepted_memory: usize = 0;
+ // What this merge already holds from the pool: its retained workspace
+ // when it has one. Seating runs within it takes nothing from others.
+ let owned = self
+ .merge_pool
+ .as_ref()
+ .map_or_else(|| reservation.size(), |pool| pool.reserved());
+ // Headroom is for others sharing the pool. With no sibling partitions
+ // and nobody else holding memory in it, seat every run that fits, as
+ // without headroom, rather than rewrite runs in an extra merge pass
+ // for no one.
+ let fan_in_contended = self
+ .fan_in_headroom
+ .as_ref()
+ .is_some_and(|headroom| headroom.is_contended(owned));
for (spill, _) in &self.sorted_spill_files {
if number_of_spills_to_read_for_current_phase >= max_spill_files
@@ -609,7 +658,20 @@ impl MultiLevelMergeBuilder {
&& number_of_spills_to_read_for_current_phase
< minimum_number_of_required_streams;
let check_headroom = self.reserve_replay_headroom && !skip_headroom;
- let admission = if check_headroom {
+ let check_fan_in_headroom = !check_headroom
+ && fan_in_contended
+ && number_of_spills_to_read_for_current_phase
+ >= minimum_number_of_required_streams;
+ let admission = if check_fan_in_headroom {
+ // Ask the pool for what this merge takes beyond its own
+ // workspace, plus as much again, then return the spare bytes.
+ match total_needed.checked_add(total_needed.saturating_sub(owned)) {
+ Some(with_headroom) => {
+ try_grow_reservation_to_at_least(reservation, with_headroom)
+ }
+ None => resources_err!("Spill merge headroom exceeds usize::MAX"),
+ }
+ } else if check_headroom {
// Ask the pool for merge buffers plus equal replay space, then
// return the spare bytes before exposing the merge stream.
match total_needed.checked_mul(2) {
@@ -677,7 +739,7 @@ impl MultiLevelMergeBuilder {
}
}
- if self.reserve_replay_headroom {
+ if self.reserve_replay_headroom || fan_in_contended {
// `total_needed` may include a rejected candidate. Keep only the
// buffers that were admitted, releasing temporary replay headroom.
reservation.shrink(reservation.size() - accepted_memory);
@@ -977,7 +1039,7 @@ mod tests {
use arrow::compute::concat_batches;
use arrow::datatypes::{DataType, Field, Int64Type, Schema};
use datafusion_execution::memory_pool::{
- GreedyMemoryPool, MemoryConsumer, MemoryPool,
+ GreedyMemoryPool, MemoryConsumer, MemoryPool, UnboundedMemoryPool,
};
use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
use datafusion_physical_expr::expressions::{Column, col};
@@ -1474,6 +1536,185 @@ mod tests {
Ok(())
}
+ /// Registers a consumer holding `size` bytes of `pool`, so a merge sharing
+ /// it sees another user and leaves headroom.
+ fn hold(pool: &Arc<dyn MemoryPool>, size: usize) -> Result<MemoryReservation> {
+ let other = MemoryConsumer::new("other consumer").register(pool);
+ other.try_grow(size)?;
+ Ok(other)
+ }
+
+ /// Headroom for others sharing `pool`, with or without sibling partitions.
+ fn headroom(pool: &Arc<dyn MemoryPool>, siblings: bool) -> FanInHeadroom {
+ FanInHeadroom {
+ pool: Arc::clone(pool),
+ siblings,
+ }
+ }
+
+ #[rstest::rstest]
+ #[case::without_headroom(false, true, false, 4, 8)]
+ #[case::headroom_without_others(true, false, false, 4, 8)]
+ #[case::headroom_with_others(true, true, false, 3, 6)]
+ #[case::headroom_with_idle_siblings(true, false, true, 3, 6)]
+ fn fan_in_headroom_limits_runs_beyond_the_minimum(
+ #[case] with_headroom: bool,
+ #[case] others: bool,
+ #[case] siblings: bool,
+ #[case] seated: usize,
+ #[case] retained_batches: usize,
+ ) -> Result<()> {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spills = (0..4)
+ .map(|value| make_sorted_spill_file(&spill_manager, &schema, vec![value]))
+ .collect::<Vec<_>>();
+ let batch_memory = spills[0].max_record_batch_memory;
+ let pool: Arc<dyn MemoryPool> =
+ Arc::new(GreedyMemoryPool::new(13 * batch_memory));
+ let other = others.then(|| hold(&pool, 1)).transpose()?;
+ let mut builder = build_merge_builder(spill_manager, schema, spills, &pool, 1)
+ .with_fan_in_headroom(with_headroom.then(|| headroom(&pool, siblings)));
+ let mut reservation = builder.reservation.new_empty();
+ let SpillFilesToMerge::Ready(spills, buffer_len) =
+ builder.get_sorted_spill_files_to_merge(1, 2, &mut reservation, false)?
+ else {
+ panic!("the minimum merge should fit the pool");
+ };
+ assert_eq!(buffer_len, 1);
+ // Each run takes two batches. With headroom, the third run needs 12
+ // batches and is seated; the fourth needs 16 and is not. When nobody
+ // else holds memory and there are no sibling partitions, all four are
+ // seated, as without headroom.
+ assert_eq!(spills.len(), seated);
+ assert_eq!(reservation.size(), retained_batches * batch_memory);
+ drop((reservation, spills, builder, other));
+ assert_eq!(pool.reserved(), 0);
+ Ok(())
+ }
+
+ /// The minimum two runs are admitted against the pool alone, even when
+ /// their buffers plus headroom would not fit.
+ #[test]
+ fn fan_in_headroom_admits_the_minimum_merge_without_headroom() -> Result<()> {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spills = (0..3)
+ .map(|value| make_sorted_spill_file(&spill_manager, &schema, vec![value]))
+ .collect::<Vec<_>>();
+ let batch_memory = spills[0].max_record_batch_memory;
+ let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(5 * batch_memory));
+ let other = hold(&pool, 1)?;
+ let mut builder = build_merge_builder(spill_manager, schema, spills, &pool, 1)
+ .with_fan_in_headroom(Some(headroom(&pool, false)));
+ let mut reservation = builder.reservation.new_empty();
+ let SpillFilesToMerge::Ready(spills, _) =
+ builder.get_sorted_spill_files_to_merge(1, 2, &mut reservation, false)?
+ else {
+ panic!("the minimum merge should fit the pool");
+ };
+ // Two runs take four batches; with headroom they would take eight. The
+ // third would take twelve.
+ assert_eq!(spills.len(), 2);
+ assert_eq!(reservation.size(), 4 * batch_memory);
+ drop((reservation, spills, builder, other));
+ assert_eq!(pool.reserved(), 0);
+ Ok(())
+ }
+
+ /// With a retained workspace `W`, seating the third run, for buffers `T`
+ /// in all, asks the parent for `2 * (T - W)` more, and nothing when
+ /// `T <= W`. Afterwards the child holds `T` and the parent is charged
+ /// `max(W, T)`.
+ #[rstest::rstest]
+ #[case::runs_within_workspace(6, 0, 0, 3)]
+ #[case::runs_below_workspace(8, 0, 0, 3)]
+ #[case::exactly_the_headroom(2, 8, 0, 3)]
+ #[case::just_below_the_headroom(2, 8, 1, 2)]
+ fn fan_in_headroom_counts_the_retained_workspace(
+ #[case] workspace_batches: usize,
+ #[case] free_batches: usize,
+ #[case] short_by_bytes: usize,
+ #[case] seated: usize,
+ ) -> Result<()> {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spills = (0..3)
+ .map(|value| make_sorted_spill_file(&spill_manager, &schema, vec![value]))
+ .collect::<Vec<_>>();
+ let batch_memory = spills[0].max_record_batch_memory;
+ let per_run =
+ get_reserved_bytes_for_record_batch_size(batch_memory, batch_memory);
+ assert_eq!(per_run, 2 * batch_memory);
+ let workspace = workspace_batches * batch_memory;
+ let free = free_batches * batch_memory - short_by_bytes;
+ let capacity = 32 * batch_memory;
+ let parent: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(capacity));
+ let merge_pool = Arc::new(MergeMemoryPool::new(
+ Arc::clone(&parent),
+ MemoryConsumer::new("merge workspace"),
+ ));
+ let pool: Arc<dyn MemoryPool> = Arc::clone(&merge_pool) as Arc<dyn MemoryPool>;
+ let mut builder = build_merge_builder(spill_manager, schema, spills, &pool, 1)
+ .with_merge_pool(Some(Arc::clone(&merge_pool)))
+ .with_fan_in_headroom(Some(headroom(&parent, false)));
+ builder.reservation.try_grow(workspace)?;
+ merge_pool.retain(workspace);
+ builder.reservation.free();
+ assert_eq!(merge_pool.reserved(), workspace);
+ let other = hold(&parent, capacity - workspace - free)?;
+
+ let mut reservation = builder.reservation.new_empty();
+ let SpillFilesToMerge::Ready(spills, _) =
+ builder.get_sorted_spill_files_to_merge(1, 2, &mut reservation, false)?
+ else {
+ panic!("the minimum merge should fit the pool");
+ };
+ assert_eq!(spills.len(), seated);
+ let total = seated * per_run;
+ assert_eq!(reservation.size(), total);
+ assert_eq!(merge_pool.reserved(), workspace.max(total));
+ assert_eq!(parent.reserved(), other.size() + workspace.max(total));
+ drop((reservation, spills, builder, other));
+ merge_pool.release_unused();
+ assert_eq!(parent.reserved(), 0);
+ Ok(())
+ }
+
+ #[test]
+ fn fan_in_headroom_beyond_usize_stops_seating_runs() -> Result<()> {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ // Three runs whose buffers fit in a usize, but whose headroom for the
+ // third does not.
+ let batch_memory = usize::MAX / 10;
+ let spills = (0..3)
+ .map(|value| SortedSpillFile {
+ max_record_batch_memory: batch_memory,
+ ..make_sorted_spill_file(&spill_manager, &schema, vec![value])
+ })
+ .collect::<Vec<_>>();
+ let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
+ let other = hold(&pool, 1)?;
+ let mut builder = build_merge_builder(spill_manager, schema, spills, &pool, 1)
+ .with_fan_in_headroom(Some(headroom(&pool, false)));
+ let mut reservation = builder.reservation.new_empty();
+ let SpillFilesToMerge::Ready(spills, _) =
+ builder.get_sorted_spill_files_to_merge(1, 2, &mut reservation, false)?
+ else {
+ panic!("the minimum merge should fit an unbounded pool");
+ };
+ assert_eq!(spills.len(), 2);
+ assert_eq!(reservation.size(), 4 * batch_memory);
+ drop((reservation, spills, builder, other));
+ assert_eq!(pool.reserved(), 0);
+ Ok(())
+ }
+
#[tokio::test]
async fn replay_headroom_still_enforces_the_actual_pool() -> Result<()> {
let env = Arc::new(RuntimeEnv::default());
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index c045ccfe38..49b9abc87f 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -9,6 +9,7 @@
use std::fmt;
use std::fmt::{Debug, Formatter};
use std::future::Future;
+use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
use parking_lot::RwLock;
@@ -310,6 +311,13 @@ struct ExternalSorter {
/// How much memory to reserve for performing in-memory sort/merges
/// prior to spilling.
sort_spill_reservation_bytes: usize,
+ /// Partitions of the owning `SortExec`, this one included, still reading
+ /// their input. Each one will need memory from the pool this sort shares.
+ partitions_reading_input: Arc<AtomicUsize>,
+ /// Whether this partition is still counted in `partitions_reading_input`.
+ /// It leaves the count when it starts its final sort, or when it is
+ /// dropped before that (its stream dropped early, or its input failed).
+ reading_input: bool,
/// Shared by the owning SortExec's partitions. Initialized on the runtime
/// that polls the sort, which may differ from the one calling `execute`.
sort_concurrency: Arc<OnceLock<SortConcurrency>>,
@@ -373,10 +381,37 @@ impl ExternalSorter {
batch_size,
sort_spill_reservation_bytes,
sort_in_place_threshold_bytes,
+ partitions_reading_input: Arc::new(AtomicUsize::new(1)),
+ reading_input: true,
sort_concurrency: Arc::new(OnceLock::new()),
})
}
+ /// Share the count of partitions still reading their input with the
+ /// other partitions of this sort. It must include this partition.
+ fn with_partitions_reading_input(mut self, reading: Arc<AtomicUsize>) -> Self {
+ self.partitions_reading_input = reading;
+ self
+ }
+
+ /// Records that this partition has read all its input, and returns whether
+ /// another partition still reads its own. Executing partitions again
+ /// without `reset_state` finds the count already at zero and keeps it
+ /// there, so those partitions keep headroom only for other consumers of
+ /// the pool; their output is the same.
+ fn finish_reading_input(&mut self) -> bool {
+ if !std::mem::replace(&mut self.reading_input, false) {
+ return false;
+ }
+ let before = self
+ .partitions_reading_input
+ .try_update(Ordering::AcqRel, Ordering::Acquire, |reading| {
+ Some(reading.saturating_sub(1))
+ })
+ .unwrap_or_else(|reading| reading);
+ before > 1
+ }
+
/// Appends an unsorted [`RecordBatch`] to `in_mem_batches`
///
/// Updates memory usage metrics, and possibly triggers spilling to disk
@@ -407,6 +442,7 @@ impl ExternalSorter {
/// 2. A combined streaming merge incorporating both in-memory
/// batches and data from spill files on disk.
async fn sort(&mut self) -> Result<SendableRecordBatchStream> {
+ let other_partitions_reading = self.finish_reading_input();
if self.spilled_before() {
// Sort `in_mem_batches` and spill it first. If there are many
// `in_mem_batches` and the memory limit is almost reached, merging
@@ -432,6 +468,21 @@ impl ExternalSorter {
.with_fetch(None)
.with_reservation(self.merge_reservation.take())
.with_merge_pool(Arc::clone(&self.merge_pool))
+ // Other partitions of this sort, still buffering input, share
+ // the pool with this merge, and may hold nothing at the moment
+ // it seats its runs. Seating every run the pool admits would
+ // leave them nothing, so keep headroom while any of them is
+ // still reading or anyone else holds memory in the pool;
+ // partitions already merging hold memory, so the second check
+ // covers them. A lone partition, or the last one to finish
+ // reading, with the pool to itself seats every run that fits:
+ // the operator consuming its output reserves nothing before
+ // its first batch, and keeping headroom for it would cost an
+ // extra merge pass whether or not it ever needs the memory.
+ .with_fan_in_headroom(
+ Arc::clone(&self.runtime.memory_pool),
+ other_partitions_reading,
+ )
.build()
} else {
// Final output needs no reserve for future spills. Return unused
@@ -1317,6 +1368,14 @@ fn sorted_copy_keeps_buffers(data_type: &DataType) -> bool {
}
}
+impl Drop for ExternalSorter {
+ /// A partition dropped before its final sort no longer reads its input,
+ /// so it must not keep its siblings' merges holding headroom for it.
+ fn drop(&mut self) {
+ self.finish_reading_input();
+ }
+}
+
impl Debug for ExternalSorter {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
f.debug_struct("ExternalSorter")
@@ -1397,6 +1456,10 @@ pub struct SortExec {
/// Batch-sort limit shared by partitions and clones.
/// `reset_state` returns a new plan with a fresh limit.
sort_concurrency: Arc<OnceLock<SortConcurrency>>,
+ /// Partitions still reading their input, shared with their sorters. Set
+ /// on the first `execute`, once the input can no longer change.
+ /// `reset_state` returns a new plan with a fresh count.
+ partitions_reading_input: Arc<OnceLock<Arc<AtomicUsize>>>,
}
impl SortExec {
@@ -1417,6 +1480,7 @@ impl SortExec {
cache: Arc::new(cache),
filter: None,
sort_concurrency: Arc::new(OnceLock::new()),
+ partitions_reading_input: Arc::new(OnceLock::new()),
}
}
@@ -1494,6 +1558,7 @@ impl SortExec {
cache: Arc::clone(&self.cache),
filter: self.filter.clone(),
sort_concurrency: Arc::clone(&self.sort_concurrency),
+ partitions_reading_input: Arc::clone(&self.partitions_reading_input),
}
}
@@ -1824,14 +1889,15 @@ impl ExecutionPlan for SortExec {
.downcast_ref::<SortExec>()
.expect("rebuilt SortExec with new children")
.clone();
- // Our dynamic filter, execution metrics and sort concurrency are the state
- // we need to reset.
+ // Our dynamic filter, execution metrics, sort concurrency and count of
+ // partitions reading input are the state we need to reset.
if let Some(filter) = &new_sort.filter {
filter.read().expr().reset();
new_sort.filter = Some(new_sort.create_filter());
}
new_sort.metrics_set = ExecutionPlanMetricsSet::new();
new_sort.sort_concurrency = Arc::new(OnceLock::new());
+ new_sort.partitions_reading_input = Arc::new(OnceLock::new());
Ok(Arc::new(new_sort))
}
@@ -1912,7 +1978,14 @@ impl ExecutionPlan for SortExec {
context.session_config().spill_compression(),
&self.metrics_set,
context.runtime_env(),
- )?;
+ )?
+ .with_partitions_reading_input(Arc::clone(
+ self.partitions_reading_input.get_or_init(|| {
+ Arc::new(AtomicUsize::new(
+ self.input.output_partitioning().partition_count(),
+ ))
+ }),
+ ));
sorter.sort_concurrency = Arc::clone(&self.sort_concurrency);
Ok(Box::pin(RecordBatchStreamAdapter::new(
self.schema(),
@@ -2115,6 +2188,7 @@ impl ExecutionPlan for SortExec {
filter,
// Execution state, initialized on the runtime polling the decoded plan.
sort_concurrency: _,
+ partitions_reading_input: _,
} = self;
let input = ctx.encode_child(input)?;
let expr = expr
diff --git a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
index fa132fb91b..53a0b4c0d8 100644
--- a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
+++ b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
@@ -3,11 +3,14 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
use super::{
- ExternalSorter, get_reserved_bytes_for_record_batch, sort_batch, sort_batch_chunked,
+ ExternalSorter, SortExec, get_reserved_bytes_for_record_batch,
+ get_reserved_bytes_for_record_batch_size, sort_batch, sort_batch_chunked,
};
+use crate::ExecutionPlan;
use crate::metrics::ExecutionPlanMetricsSet;
use crate::spill::get_record_batch_memory_size;
use crate::spill::spill_manager::GetSlicedSize;
+use crate::test::TestMemoryExec;
use arrow::array::{
ArrayRef, Decimal128Array, DictionaryArray, FixedSizeBinaryArray, Int8Array,
Int32Array, Int64Array, StringArray, StringViewArray,
@@ -17,6 +20,8 @@ use arrow::datatypes::{DataType, Field, Int8Type, Schema};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::{ExecutionOptions, SpillCompression};
use datafusion_common::{DataFusionError, Result};
+use datafusion_execution::TaskContext;
+use datafusion_execution::config::SessionConfig;
use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
use datafusion_execution::memory_pool::{
GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation,
@@ -26,6 +31,7 @@ use datafusion_physical_expr::expressions::{CastExpr, Column};
use datafusion_physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr};
use futures::TryStreamExt;
use std::fmt::{Display, Formatter};
+use std::sync::atomic::AtomicUsize;
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -1428,3 +1434,208 @@ async fn test_single_batch_spill_returns_live_workspace_loan_on_drop() -> Result
assert_released(&pool, &runtime).await;
Ok(())
}
+
+/// The final merge of a spilled sort shares the pool with sibling partitions
+/// still sorting their input. Seating every run the pool admits leaves them
+/// nothing, so whatever the merge takes beyond its own workspace, it must leave
+/// as much free: while another consumer holds memory, and while sibling
+/// partitions still read their input even if they hold nothing at the moment.
+#[rstest::rstest]
+#[case::other_consumer_holds_memory(false)]
+#[case::idle_sibling_partitions(true)]
+#[tokio::test]
+async fn test_final_spilled_merge_leaves_headroom(#[case] siblings: bool) -> Result<()> {
+ let fixture = fixture()?;
+ let schema = fixture.parent.schema();
+ let capacity = 16 * fixture.capacity;
+ let pool = AdjustablePool::new(capacity);
+ // Spill under the fixture's limit, then merge with room for every run.
+ pool.set_limit(fixture.capacity);
+ let (sorter, runtime) = sorter(&fixture, Arc::clone(&pool))?;
+ let mut sorter = sorter.with_partitions_reading_input(Arc::new(AtomicUsize::new(
+ 1 + usize::from(siblings),
+ )));
+ for batch in &fixture.batches {
+ sorter.insert_batch(batch.clone()).await?;
+ }
+ if !sorter.in_mem_batches.is_empty() {
+ sorter.sort_and_spill_in_mem_batches().await?;
+ }
+ // What the merge reserves to read each run with two buffers.
+ let needs: Vec<usize> = sorter
+ .finished_spill_files
+ .iter()
+ .map(|spill| {
+ let size = spill.max_record_batch_memory;
+ 2 * get_reserved_bytes_for_record_batch_size(size, size)
+ })
+ .collect();
+ assert!(needs.len() >= 3, "need three runs, got {}", needs.len());
+ let all: usize = needs.iter().sum();
+ let smallest = *needs.iter().min().unwrap();
+ pool.set_limit(capacity);
+
+ let mut stream = sorter.sort().await?;
+ drop(sorter);
+ // The merge already holds its workspace. Leave it room to seat every
+ // run, but not to seat them all and leave as much free.
+ let workspace = pool.reserved();
+ let free = (2 * all)
+ .checked_sub(2 * workspace + smallest)
+ .expect("the runs must need more than the workspace");
+ assert!(all <= free + workspace);
+ // Either another consumer holds the rest of the pool, or idle siblings
+ // hold nothing and the pool is only as large as the merge may use.
+ let contender =
+ MemoryConsumer::new("sibling partition").register(&runtime.memory_pool);
+ let limit = if siblings {
+ pool.set_limit(workspace + free);
+ workspace + free
+ } else {
+ contender.try_grow(capacity - workspace - free)?;
+ capacity
+ };
+
+ let first = stream
+ .try_next()
+ .await?
+ .expect("the merge must produce output");
+ let taken = (pool.reserved() - contender.size()).saturating_sub(workspace);
+ assert!(
+ limit - pool.reserved() >= taken,
+ "the final merge takes {taken} bytes beyond its workspace and leaves {} free",
+ limit - pool.reserved()
+ );
+ drop(contender);
+
+ let mut output = vec![first];
+ while let Some(batch) = stream.try_next().await? {
+ output.push(batch);
+ }
+ assert_eq!(
+ concat_batches(&schema, &output)?,
+ sort_batch(&fixture.parent, &fixture.ordering, None)?
+ );
+ drop(stream);
+ assert_released(&pool, &runtime).await;
+ Ok(())
+}
+
+/// What partition 1 does before partition 0 runs.
+#[derive(Debug, Clone, Copy)]
+enum Sibling {
+ /// Never executed, so it still counts as reading its input.
+ Idle,
+ /// Read all its (empty) input.
+ Finished,
+ /// Executed, then its stream dropped without being polled.
+ Dropped,
+}
+
+/// Whether a `SortExec` keeps fan-in headroom follows its partitions still
+/// reading input. The pool fits every run's buffers but not the headroom on
+/// top, so with headroom the final merge seats fewer runs and rewrites rows in
+/// an extra pass. Partition 0 runs alone, so nothing else ever holds pool
+/// memory and the outcome is deterministic. In the last two cases its sibling
+/// has read all its input or has been dropped before it could, so there is
+/// nobody to keep headroom for.
+#[rstest::rstest]
+#[case::lone_partition(1, Sibling::Idle, false)]
+#[case::idle_sibling(2, Sibling::Idle, true)]
+#[case::finished_sibling(2, Sibling::Finished, false)]
+#[case::dropped_sibling(2, Sibling::Dropped, false)]
+#[tokio::test]
+async fn test_sort_exec_keeps_headroom_only_while_siblings_read_input(
+ #[case] input_partitions: usize,
+ #[case] sibling: Sibling,
+ #[case] rewrites_rows: bool,
+) -> Result<()> {
+ let (sort, context, total_rows) = headroom_sort_exec(input_partitions)?;
+ match sibling {
+ Sibling::Idle => {}
+ Sibling::Finished => {
+ let output: Vec<RecordBatch> =
+ sort.execute(1, Arc::clone(&context))?.try_collect().await?;
+ assert!(output.is_empty());
+ }
+ Sibling::Dropped => drop(sort.execute(1, Arc::clone(&context))?),
+ }
+ let output: Vec<RecordBatch> = sort.execute(0, context)?.try_collect().await?;
+
+ assert_eq!(
+ output.iter().map(RecordBatch::num_rows).sum::<usize>(),
+ total_rows
+ );
+ let spilled_rows = sort.metrics().unwrap().spilled_rows().unwrap();
+ if rewrites_rows {
+ assert!(spilled_rows > total_rows, "{spilled_rows} <= {total_rows}");
+ } else {
+ assert_eq!(spilled_rows, total_rows);
+ }
+ Ok(())
+}
+
+/// Executing a lone partition again without `reset_state` finds the count of
+/// partitions reading input at zero. The count must stay there instead of
+/// wrapping, or the third run would keep headroom for nobody.
+#[tokio::test]
+async fn test_sort_exec_rerun_without_reset_state_keeps_no_headroom() -> Result<()> {
+ let (sort, context, total_rows) = headroom_sort_exec(1)?;
+ for run in 1..=3 {
+ let output: Vec<RecordBatch> =
+ sort.execute(0, Arc::clone(&context))?.try_collect().await?;
+ assert_eq!(
+ output.iter().map(RecordBatch::num_rows).sum::<usize>(),
+ total_rows
+ );
+ let spilled_rows = sort.metrics().unwrap().spilled_rows().unwrap();
+ assert_eq!(spilled_rows, run * total_rows, "run {run}");
+ }
+ Ok(())
+}
+
+/// A `SortExec` over `input_partitions` partitions with all rows in partition
+/// 0, and a context whose pool fits every run's buffers but not the headroom
+/// on top. Returns the sort, the context, and the number of rows.
+fn headroom_sort_exec(
+ input_partitions: usize,
+) -> Result<(SortExec, Arc<TaskContext>, usize)> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int64, false),
+ Field::new("payload", DataType::Utf8, false),
+ ]));
+ let (batches, rows) = (64, 1024);
+ let batches = (0..batches)
+ .map(|batch| {
+ let keys = (0..rows).map(|row| ((batch * rows + row) * 2654435761) % 1000003);
+ let payload = (0..rows).map(|row| format!("{row:0120}"));
+ let columns: Vec<ArrayRef> = vec![
+ Arc::new(Int64Array::from_iter_values(keys)),
+ Arc::new(StringArray::from_iter_values(payload)),
+ ];
+ Ok(RecordBatch::try_new(Arc::clone(&schema), columns)?)
+ })
+ .collect::<Result<Vec<_>>>()?;
+ let total_rows: usize = batches.iter().map(RecordBatch::num_rows).sum();
+ let mut partitions = vec![batches];
+ partitions.resize(input_partitions, vec![]);
+ let input = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;
+ let ordering: LexOrdering = [PhysicalSortExpr::new_default(Arc::new(Column::new(
+ "key", 0,
+ )))]
+ .into();
+ let sort = SortExec::new(ordering, input).with_preserve_partitioning(true);
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(3500 * 1024)))
+ .build_arc()?;
+ let context = Arc::new(
+ TaskContext::default()
+ .with_session_config(
+ SessionConfig::new()
+ .with_batch_size(1024)
+ .with_sort_spill_reservation_bytes(128 * 1024),
+ )
+ .with_runtime(runtime),
+ );
+ Ok((sort, context, total_rows))
+}
diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs
index 1668a55ac9..98f4579067 100644
--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs
+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs
@@ -6,7 +6,7 @@
//! This is an order-preserving merge.
use crate::metrics::BaselineMetrics;
-use crate::sorts::multi_level_merge::MultiLevelMergeBuilder;
+use crate::sorts::multi_level_merge::{FanInHeadroom, MultiLevelMergeBuilder};
use crate::sorts::{
merge::SortPreservingMergeStream,
stream::{FieldCursorStream, RowCursorStream},
@@ -87,6 +87,9 @@ pub struct StreamingMergeBuilder<'a> {
merge_pool: Option<Arc<MergeMemoryPool>>,
/// Leave memory for the aggregate consuming the merged spill rows.
reserve_replay_headroom: bool,
+ /// Leave memory for others sharing the execution pool when seating more
+ /// than the minimum two runs.
+ fan_in_headroom: Option<FanInHeadroom>,
min_spill_batch_rows: Option<usize>,
enable_round_robin_tie_breaker: bool,
}
@@ -166,6 +169,23 @@ impl<'a> StreamingMergeBuilder<'a> {
self
}
+ /// While `siblings` is true or another consumer holds memory in `pool`,
+ /// seat spill runs beyond the minimum two only if the pool can admit their
+ /// buffers plus as much again as they exceed the retained workspace.
+ /// Release the headroom before merging.
+ ///
+ /// `pool` is the execution pool: the parent of the merge pool when one is
+ /// set, not the merge pool itself. `siblings` says other partitions share
+ /// that pool even when they hold nothing at the moment.
+ pub(crate) fn with_fan_in_headroom(
+ mut self,
+ pool: Arc<dyn MemoryPool>,
+ siblings: bool,
+ ) -> Self {
+ self.fan_in_headroom = Some(FanInHeadroom { pool, siblings });
+ self
+ }
+
/// Limit intermediate rewrites when the retained buffer reservation can
/// also admit the final replay pass with its headroom.
/// `min_spill_batch_rows` is the minimum largest-batch row count across input
@@ -214,6 +234,7 @@ impl<'a> StreamingMergeBuilder<'a> {
reservation,
merge_pool,
reserve_replay_headroom,
+ fan_in_headroom,
min_spill_batch_rows,
fetch,
expressions,
@@ -257,6 +278,7 @@ impl<'a> StreamingMergeBuilder<'a> {
)
.with_merge_pool(merge_pool)
.with_replay_headroom(reserve_replay_headroom)
+ .with_fan_in_headroom(fan_in_headroom)
.with_intermediate_merge_sizing(min_spill_batch_rows)
.with_max_batch_bytes(max_batch_bytes)
.create_spillable_merge_stream());