Skip to content

Apache DataFusion SQL Query Engine

perf: avoid unnecessary aggregate spill rewrites (#25428)

Chao Sunf029e2b1acf0
diff --git a/datafusion/physical-plan/src/aggregates/spill.rs b/datafusion/physical-plan/src/aggregates/spill.rs
index a1d52382de..887be41855 100644
--- a/datafusion/physical-plan/src/aggregates/spill.rs
+++ b/datafusion/physical-plan/src/aggregates/spill.rs
@@ -116,6 +116,8 @@ pub(super) struct AggregateSpill {
spill_manager: SpillManager,
/// Spill runs waiting to be merged, all sorted by `spill_expr`.
spills: Vec<SortedSpillFile>,
+ /// Minimum, across original runs, of each run's largest batch row count.
+ min_spill_batch_rows: usize,
/// Describes this stream's spill requests, and prefixes its internal errors.
label: &'static str,
}
@@ -200,6 +202,7 @@ impl AggregateSpill {
spill_expr,
spill_manager,
spills: vec![],
+ min_spill_batch_rows: batch_size,
label,
})
}
@@ -219,6 +222,7 @@ impl AggregateSpill {
return Ok(());
};
+ let max_batch_rows = state_batch.num_rows().min(self.batch_size);
let sorted_iter = IncrementalSortIterator::new(
state_batch,
self.spill_expr.clone(),
@@ -239,6 +243,7 @@ impl AggregateSpill {
file,
max_record_batch_memory,
});
+ self.min_spill_batch_rows = self.min_spill_batch_rows.min(max_batch_rows);
Ok(())
}
@@ -259,6 +264,7 @@ impl AggregateSpill {
spill_expr,
spill_manager,
spills,
+ min_spill_batch_rows,
label: _,
} = self;
@@ -276,6 +282,7 @@ impl AggregateSpill {
.with_batch_size(batch_size)
.with_reservation(merge_reservation)
.with_replay_headroom()
+ .with_intermediate_merge_sizing(Some(min_spill_batch_rows))
.build()?;
let replay = OrderedFinalAggregateStream::new_with_input_and_metrics(
&replay_agg,
diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge.rs b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
index c1fe893e0d..28a8000265 100644
--- a/datafusion/physical-plan/src/sorts/multi_level_merge.rs
+++ b/datafusion/physical-plan/src/sorts/multi_level_merge.rs
@@ -159,6 +159,8 @@ pub(crate) struct MultiLevelMergeBuilder {
merge_pool: Option<Arc<MergeMemoryPool>>,
/// Leave memory for the aggregate consuming the merged spill rows.
reserve_replay_headroom: bool,
+ /// Avoid merging more runs than needed before the final replay pass.
+ size_intermediate_merges: bool,
fetch: Option<usize>,
enable_round_robin_tie_breaker: bool,
}
@@ -199,6 +201,7 @@ impl MultiLevelMergeBuilder {
reservation,
merge_pool: None,
reserve_replay_headroom: false,
+ size_intermediate_merges: false,
enable_round_robin_tie_breaker,
fetch,
}
@@ -216,6 +219,17 @@ impl MultiLevelMergeBuilder {
self
}
+ pub(super) fn with_intermediate_merge_sizing(
+ mut self,
+ min_spill_batch_rows: Option<usize>,
+ ) -> Self {
+ // Short inputs can produce larger batches when merged, reducing the next
+ // pass's fan-in. Fetch can also leave intermediate runs with short batches.
+ self.size_intermediate_merges = self.fetch.is_none()
+ && min_spill_batch_rows.is_some_and(|rows| rows >= self.batch_size);
+ self
+ }
+
pub(crate) fn create_spillable_merge_stream(self) -> SendableRecordBatchStream {
Box::pin(RecordBatchStreamAdapter::new(
Arc::clone(&self.schema),
@@ -275,9 +289,9 @@ impl MultiLevelMergeBuilder {
}
// Need to sort to a spill file
- let Some((spill_file, max_record_batch_memory)) = self
+ let Some((spill_file, max_record_batch_memory, max_batch_rows)) = self
.spill_manager
- .spill_record_batch_stream_and_return_max_batch_memory(
+ .spill_record_batch_stream_and_return_max_batch_stats(
&mut stream,
"MultiLevelMergeBuilder intermediate spill",
)
@@ -286,6 +300,10 @@ impl MultiLevelMergeBuilder {
continue;
};
+ // Offset overflow can flush partial batches even without a fetch limit.
+ // Keep sizing disabled once a run cannot fill its output batch limit.
+ self.size_intermediate_merges &= max_batch_rows >= batch_size_limit;
+
// Add the spill file paired with the batch-size limit of the merge that
// produced it: if that merge consumed a shrunk (skew-resolved) run, its
// output was capped and this intermediate run is likewise capped, so a
@@ -373,7 +391,7 @@ impl MultiLevelMergeBuilder {
let minimum_number_of_required_streams =
2_usize.saturating_sub(self.sorted_streams.len());
- let (sorted_spill_files, buffer_size) = match self
+ let (mut sorted_spill_files, buffer_size) = match self
.get_sorted_spill_files_to_merge(
2,
// we must have at least 2 streams to merge
@@ -393,6 +411,47 @@ impl MultiLevelMergeBuilder {
}
};
+ if self.size_intermediate_merges
+ && self.sorted_streams.is_empty()
+ && !self.sorted_spill_files.is_empty()
+ {
+ // The next admission starts with two read buffers per run.
+ // Fit both those buffers and any replay headroom in the
+ // existing grant: another partition may take every spare
+ // byte while this intermediate pass writes to disk.
+ let final_fan_in = sorted_spill_files.len() * buffer_size / 2;
+ let final_fan_in = if self.reserve_replay_headroom {
+ final_fan_in / 2
+ } else {
+ final_fan_in
+ };
+ let needed = self.sorted_spill_files.len() + sorted_spill_files.len()
+ - final_fan_in
+ + 1;
+ if needed < sorted_spill_files.len() {
+ let (first, first_limit) = &sorted_spill_files[0];
+ // A wider pending run or a smaller output batch limit can
+ // change the next pass's fan-in and add intermediate work.
+ // Apply this sizing heuristic only to uniform run budgets.
+ let uniform_runs = sorted_spill_files
+ .iter()
+ .chain(&self.sorted_spill_files)
+ .all(|(file, limit)| {
+ file.max_record_batch_memory
+ == first.max_record_batch_memory
+ && limit == first_limit
+ });
+ if uniform_runs {
+ let unused = sorted_spill_files.split_off(needed);
+ self.sorted_spill_files.splice(0..0, unused);
+ // Keep the grant through intermediate EOF and the
+ // spill write, then transfer it to the next admission.
+ // This does not reserve additional pool memory.
+ self.reservation = memory_reservation.take();
+ }
+ }
+ }
+
// Don't account for existing streams memory
// as we are not holding the memory for them
let mut sorted_streams = mem::take(&mut self.sorted_streams);
@@ -751,7 +810,7 @@ impl MultiLevelMergeBuilder {
let result = self
.spill_manager
- .spill_record_batch_stream_and_return_max_batch_memory(
+ .spill_record_batch_stream_and_return_max_batch_stats(
&mut halved,
"MultiLevelMergeBuilder split skewed spill",
)
@@ -759,7 +818,7 @@ impl MultiLevelMergeBuilder {
reservation.free();
- let Some((file, new_max)) = result else {
+ let Some((file, new_max, new_max_batch_rows)) = result else {
return internal_err!("re-spilling a skewed spill file produced no data");
};
@@ -787,6 +846,7 @@ impl MultiLevelMergeBuilder {
// larger output batch formed by concatenating those rows.
1
};
+ self.size_intermediate_merges &= new_max_batch_rows >= new_batch_size_limit;
// Push the re-spilled (smaller) file and swap it back into `index`, undoing
// the swap-to-back above so the order is preserved.
@@ -906,6 +966,10 @@ impl RecordBatchStream for StreamAttachedReservation {
}
}
+#[cfg(test)]
+#[path = "multi_level_merge/intermediate_sizing_tests.rs"]
+mod intermediate_sizing_tests;
+
#[cfg(test)]
mod tests {
use super::*;
@@ -923,11 +987,14 @@ mod tests {
ExecutionPlanMetricsSet, SpillMetrics,
};
- fn test_schema() -> SchemaRef {
+ pub(super) fn test_schema() -> SchemaRef {
Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)]))
}
- fn build_spill_manager(env: &Arc<RuntimeEnv>, schema: &SchemaRef) -> SpillManager {
+ pub(super) fn build_spill_manager(
+ env: &Arc<RuntimeEnv>,
+ schema: &SchemaRef,
+ ) -> SpillManager {
SpillManager::new(
Arc::clone(env),
SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0),
@@ -937,7 +1004,7 @@ mod tests {
/// Spill `values` (which must already be sorted) as a single sorted run and
/// return it as a `SortedSpillFile` carrying its recorded largest-batch memory.
- fn make_sorted_spill_file(
+ pub(super) fn make_sorted_spill_file(
spill_manager: &SpillManager,
schema: &SchemaRef,
values: Vec<i64>,
@@ -961,7 +1028,7 @@ mod tests {
}
}
- fn build_merge_builder(
+ pub(super) fn build_merge_builder(
spill_manager: SpillManager,
schema: SchemaRef,
sorted_spill_files: Vec<SortedSpillFile>,
@@ -985,6 +1052,150 @@ mod tests {
)
}
+ #[rstest::rstest]
+ #[case::full_batches(8, 128, 6)]
+ #[case::replay_headroom_limits_sizing(9, 128, 7)]
+ #[case::unchanged_selection(13, 128, 7)]
+ #[case::short_batches(9, 64, 7)]
+ #[tokio::test]
+ async fn intermediate_merge_sizing_preserves_final_output(
+ #[values(false, true)] size_intermediate_merges: bool,
+ #[case] runs: usize,
+ #[case] rows: usize,
+ #[case] sized_intermediate_inputs: usize,
+ ) -> Result<()> {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();
+ let files: Vec<_> = (0..runs)
+ .map(|run| {
+ make_sorted_spill_file(
+ &spill_manager,
+ &schema,
+ (0..rows).map(|row| (row * runs + run) as i64).collect(),
+ )
+ })
+ .collect();
+ let batch_memory = files[0].max_record_batch_memory;
+ // Ordinary admission seats seven inputs with read-ahead and equal replay
+ // headroom. The retained buffer grant can admit three final inputs plus
+ // their replay headroom, so eight runs need six merged inputs. Nine or
+ // thirteen runs require all seven and exercise the unchanged selection.
+ // Short input batches can grow when merged, so they must keep the original
+ // selection to avoid adding a pass with the larger output batch budget.
+ let pool: Arc<dyn MemoryPool> =
+ Arc::new(GreedyMemoryPool::new(8 * 7 * batch_memory));
+ let mut builder =
+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, 128)
+ .with_replay_headroom(true);
+ if size_intermediate_merges {
+ builder = builder.with_intermediate_merge_sizing(Some(rows));
+ }
+ let stream = builder.create_spillable_merge_stream();
+ let batches: Vec<RecordBatch> = stream.try_collect().await?;
+ let merged = concat_batches(&schema, &batches)?;
+ let expected = Int64Array::from_iter_values(0..(runs * rows) as i64);
+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);
+ let intermediate_inputs = if size_intermediate_merges {
+ sized_intermediate_inputs
+ } else {
+ 7
+ };
+ assert_eq!(spilled_rows.value(), (runs + intermediate_inputs) * rows);
+ assert_eq!(pool.reserved(), 0);
+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);
+ assert_eq!(env.disk_manager.used_disk_space(), 0);
+ Ok(())
+ }
+
+ #[rstest::rstest]
+ #[tokio::test]
+ async fn intermediate_merge_sizing_preserves_heterogeneous_runs(
+ #[values(false, true)] size_intermediate_merges: bool,
+ ) -> Result<()> {
+ use arrow::array::StringArray;
+ use arrow::buffer::{Buffer, OffsetBuffer};
+
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("x", DataType::Int64, false),
+ Field::new("payload", DataType::Utf8, false),
+ ]));
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();
+ let rows = 128;
+ let mut files = Vec::new();
+ for run in 0..8 {
+ let width = if run == 6 { 256 } else { 16 };
+ let payload = StringArray::new(
+ OffsetBuffer::new(
+ (0..=rows)
+ .map(|row| (row * width) as i32)
+ .collect::<Vec<_>>()
+ .into(),
+ ),
+ Buffer::from(vec![b'x'; rows * width]),
+ None,
+ );
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int64Array::from_iter_values(
+ (run * rows..(run + 1) * rows).map(|value| value as i64),
+ )),
+ Arc::new(payload),
+ ],
+ )?;
+ let (file, max_record_batch_memory) = spill_manager
+ .spill_record_batch_iter_and_return_max_batch_memory(
+ std::iter::once(Ok(batch)),
+ "heterogeneous input run",
+ )?
+ .unwrap();
+ files.push(SortedSpillFile {
+ file,
+ max_record_batch_memory,
+ });
+ }
+ let small = files[0].max_record_batch_memory;
+ let wide = files[6].max_record_batch_memory;
+ // Admit six small inputs first, then the wide input and one small input.
+ // Trimming the first merge to three wrongly assumes its fan-in will also
+ // fit the wider suffix, adding a later rewrite of the small runs.
+ let pool: Arc<dyn MemoryPool> =
+ Arc::new(GreedyMemoryPool::new(8 * (wide + small + small / 2)));
+ let builder =
+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, rows)
+ .with_replay_headroom(true)
+ .with_intermediate_merge_sizing(size_intermediate_merges.then_some(rows));
+ let batches: Vec<RecordBatch> = builder
+ .create_spillable_merge_stream()
+ .try_collect()
+ .await?;
+ let merged = concat_batches(&schema, &batches)?;
+ let expected = Int64Array::from_iter_values(0..(8 * rows) as i64);
+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);
+ let small_payload = "x".repeat(16);
+ let wide_payload = "x".repeat(256);
+ let expected_payload = StringArray::from_iter_values((0..8).flat_map(|run| {
+ std::iter::repeat_n(
+ if run == 6 {
+ &wide_payload
+ } else {
+ &small_payload
+ },
+ rows,
+ )
+ }));
+ assert_eq!(merged.column(1).as_string::<i32>(), &expected_payload);
+ assert_eq!(spilled_rows.value(), 16 * rows);
+ assert_eq!(pool.reserved(), 0);
+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);
+ assert_eq!(env.disk_manager.used_disk_space(), 0);
+ Ok(())
+ }
+
/// Two sorted runs whose largest batches are too big to both
/// be seated in the merge budget at once are re-spilled (halved) until they
/// fit, and the merge then completes with fully sorted, complete output.
diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rs b/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rs
new file mode 100644
index 0000000000..4841d4b59f
--- /dev/null
+++ b/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rs
@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::tests::{
+ build_merge_builder, build_spill_manager, make_sorted_spill_file, test_schema,
+};
+use super::*;
+use arrow::array::{AsArray, Int64Array};
+use arrow::compute::concat_batches;
+use arrow::datatypes::Int64Type;
+use datafusion_execution::memory_pool::{
+ GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool,
+};
+use datafusion_execution::runtime_env::RuntimeEnv;
+use std::fmt::Display;
+use std::sync::Mutex;
+
+#[derive(Debug, Clone, Copy)]
+enum PressurePoint {
+ ReplayHeadroom,
+ IntermediateEof,
+}
+
+/// Schedule another consumer immediately when the merge returns memory to the
+/// shared pool. Taking only available bytes models an ordinary competing
+/// reservation while making both release boundaries deterministic.
+#[derive(Debug)]
+struct CompetingMemoryPool {
+ inner: Arc<dyn MemoryPool>,
+ contender: MemoryReservation,
+ capacity: usize,
+ contender_target: usize,
+ pressure_point: PressurePoint,
+ active: Mutex<bool>,
+}
+
+impl CompetingMemoryPool {
+ fn new(
+ capacity: usize,
+ contender_target: usize,
+ pressure_point: PressurePoint,
+ ) -> Self {
+ let inner: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(capacity));
+ let contender = MemoryConsumer::new("competing partition").register(&inner);
+ Self {
+ inner,
+ contender,
+ capacity,
+ contender_target,
+ pressure_point,
+ active: Mutex::new(false),
+ }
+ }
+}
+
+impl Display for CompetingMemoryPool {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ write!(f, "CompetingMemoryPool({})", self.inner)
+ }
+}
+
+impl MemoryPool for CompetingMemoryPool {
+ fn name(&self) -> &str {
+ "competing"
+ }
+
+ fn grow(&self, reservation: &MemoryReservation, additional: usize) {
+ self.inner.grow(reservation, additional);
+ }
+
+ fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
+ self.inner.shrink(reservation, shrink);
+ if shrink == 0 {
+ return;
+ }
+ let mut active = self.active.lock().unwrap();
+ *active |= match self.pressure_point {
+ PressurePoint::ReplayHeadroom => reservation.size() > 0,
+ PressurePoint::IntermediateEof => reservation.size() == 0,
+ };
+ if *active {
+ let additional = (self.contender_target - self.contender.size())
+ .min(self.capacity - self.inner.reserved());
+ self.contender.try_grow(additional).unwrap();
+ }
+ }
+
+ fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {
+ self.inner.try_grow(reservation, additional)
+ }
+
+ fn reserved(&self) -> usize {
+ self.inner.reserved()
+ }
+
+ fn memory_limit(&self) -> MemoryLimit {
+ self.inner.memory_limit()
+ }
+}
+
+#[rstest::rstest]
+#[case::headroom_release(PressurePoint::ReplayHeadroom)]
+#[case::intermediate_eof(PressurePoint::IntermediateEof)]
+#[tokio::test]
+async fn intermediate_sizing_does_not_add_rewrites_under_contention(
+ #[case] pressure_point: PressurePoint,
+ #[values(2, 3)] final_fan_in: usize,
+ #[values(8, 9)] runs: usize,
+) -> Result<()> {
+ const ROWS: usize = 128;
+ let mut rewrites = Vec::new();
+ for size_intermediate_merges in [false, true] {
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();
+ let files: Vec<_> = (0..runs)
+ .map(|run| {
+ make_sorted_spill_file(
+ &spill_manager,
+ &schema,
+ (0..ROWS).map(|row| (row * runs + run) as i64).collect(),
+ )
+ })
+ .collect();
+ let batch_memory = files[0].max_record_batch_memory;
+ // Seven runs initially fit with read-ahead and equal replay headroom.
+ // The competing partition takes released bytes until only two or three
+ // runs would fit a fresh admission. Both runs use the same release hook.
+ let competing_pool = Arc::new(CompetingMemoryPool::new(
+ 8 * 7 * batch_memory,
+ 8 * (7 - final_fan_in) * batch_memory,
+ pressure_point,
+ ));
+ let pool = Arc::clone(&competing_pool) as Arc<dyn MemoryPool>;
+ let builder =
+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, ROWS)
+ .with_replay_headroom(true)
+ .with_intermediate_merge_sizing(size_intermediate_merges.then_some(ROWS));
+ let mut stream = builder.create_spillable_merge_stream();
+ let first = stream.try_next().await?.expect("nonempty merge");
+ if matches!(pressure_point, PressurePoint::IntermediateEof)
+ && size_intermediate_merges
+ && runs == 8
+ {
+ // Sizing these eight runs retains the admitted buffers across the
+ // intermediate EOF. The contender cannot take them before the final
+ // merge is admitted, so this boundary must not have fired yet.
+ assert_eq!(competing_pool.contender.size(), 0);
+ } else {
+ assert_eq!(
+ competing_pool.contender.size(),
+ competing_pool.contender_target
+ );
+ }
+ let mut batches = vec![first];
+ batches.extend(stream.try_collect::<Vec<_>>().await?);
+ let merged = concat_batches(&schema, &batches)?;
+ let expected = Int64Array::from_iter_values(0..(runs * ROWS) as i64);
+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);
+ rewrites.push(spilled_rows.value() - runs * ROWS);
+ assert_eq!(
+ competing_pool.contender.size(),
+ competing_pool.contender_target
+ );
+ assert_eq!(pool.reserved(), competing_pool.contender.size());
+ competing_pool.contender.free();
+ assert_eq!(pool.reserved(), 0);
+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);
+ assert_eq!(env.disk_manager.used_disk_space(), 0);
+ }
+ assert_eq!(
+ rewrites[0],
+ if runs == 9 && final_fan_in == 2 { 9 } else { 7 } * ROWS
+ );
+ assert!(
+ rewrites[1] <= rewrites[0],
+ "sizing rewrote {} rows versus {} without sizing under {pressure_point:?}",
+ rewrites[1],
+ rewrites[0],
+ );
+ if runs == 8 {
+ assert_eq!(rewrites[1], 6 * ROWS);
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn sized_merge_retains_reservation_until_builder_drop() -> Result<()> {
+ const RUNS: usize = 8;
+ const ROWS: usize = 128;
+ let env = Arc::new(RuntimeEnv::default());
+ let schema = test_schema();
+ let spill_manager = build_spill_manager(&env, &schema);
+ let files: Vec<_> = (0..RUNS)
+ .map(|run| {
+ make_sorted_spill_file(
+ &spill_manager,
+ &schema,
+ (0..ROWS).map(|row| (row * RUNS + run) as i64).collect(),
+ )
+ })
+ .collect();
+ let batch_memory = files[0].max_record_batch_memory;
+ let competing_pool = Arc::new(CompetingMemoryPool::new(
+ 8 * 7 * batch_memory,
+ 8 * 5 * batch_memory,
+ PressurePoint::ReplayHeadroom,
+ ));
+ let pool = Arc::clone(&competing_pool) as Arc<dyn MemoryPool>;
+ let mut builder =
+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, ROWS)
+ .with_replay_headroom(true)
+ .with_intermediate_merge_sizing(Some(ROWS));
+ let MergeStep::Stream { stream, .. } =
+ builder.merge_sorted_runs_within_mem_limit(false)?
+ else {
+ panic!("six inputs should be selected for the intermediate merge");
+ };
+ assert_eq!(builder.sorted_spill_files.len(), 2);
+ let retained = 4 * 7 * batch_memory;
+ assert_eq!(builder.reservation.size(), retained);
+ assert_eq!(pool.reserved(), retained + competing_pool.contender.size());
+
+ // The intermediate stream may finish or be cancelled before the builder
+ // starts final admission. Its drop must leave the grant with the builder.
+ drop(stream);
+ assert_eq!(builder.reservation.size(), retained);
+ assert_eq!(pool.reserved(), retained + competing_pool.contender.size());
+ drop(builder);
+ assert_eq!(pool.reserved(), competing_pool.contender.size());
+ competing_pool.contender.free();
+ assert_eq!(pool.reserved(), 0);
+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);
+ assert_eq!(env.disk_manager.used_disk_space(), 0);
+ Ok(())
+}
diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs
index 61bb7742ee..199dceac8c 100644
--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs
+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs
@@ -98,6 +98,7 @@ pub struct StreamingMergeBuilder<'a> {
merge_pool: Option<Arc<MergeMemoryPool>>,
/// Leave memory for the aggregate consuming the merged spill rows.
reserve_replay_headroom: bool,
+ min_spill_batch_rows: Option<usize>,
enable_round_robin_tie_breaker: bool,
}
@@ -170,6 +171,19 @@ impl<'a> StreamingMergeBuilder<'a> {
self
}
+ /// Limit intermediate rewrites when the retained buffer reservation can
+ /// also admit the final replay pass with its headroom.
+ /// `min_spill_batch_rows` is the minimum largest-batch row count across input
+ /// runs. A run without a full batch disables sizing for the whole replay,
+ /// since merging short batches can increase the intermediate run's budget.
+ pub(crate) fn with_intermediate_merge_sizing(
+ mut self,
+ min_spill_batch_rows: Option<usize>,
+ ) -> Self {
+ self.min_spill_batch_rows = min_spill_batch_rows;
+ self
+ }
+
/// See [SortPreservingMergeExec::with_round_robin_repartition] for more
/// information.
///
@@ -204,6 +218,7 @@ impl<'a> StreamingMergeBuilder<'a> {
reservation,
merge_pool,
reserve_replay_headroom,
+ min_spill_batch_rows,
fetch,
expressions,
enable_round_robin_tie_breaker,
@@ -246,6 +261,7 @@ impl<'a> StreamingMergeBuilder<'a> {
)
.with_merge_pool(merge_pool)
.with_replay_headroom(reserve_replay_headroom)
+ .with_intermediate_merge_sizing(min_spill_batch_rows)
.create_spillable_merge_stream());
}
diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rs
index 389fafb985..9c07602fab 100644
--- a/datafusion/physical-plan/src/spill/spill_manager.rs
+++ b/datafusion/physical-plan/src/spill/spill_manager.rs
@@ -140,29 +140,32 @@ impl SpillManager {
Ok(file.map(|f| (f, max_record_batch_size)))
}
- /// Spill a stream of `RecordBatch`es to disk and return the spill file and the size of the largest batch in memory
- pub(crate) async fn spill_record_batch_stream_and_return_max_batch_memory(
+ /// Spill a stream of `RecordBatch`es and return the file, maximum batch memory,
+ /// and maximum batch row count.
+ pub(crate) async fn spill_record_batch_stream_and_return_max_batch_stats(
&self,
stream: &mut SendableRecordBatchStream,
request_description: &str,
- ) -> Result<Option<(Arc<dyn SpillFile>, usize)>> {
+ ) -> Result<Option<(Arc<dyn SpillFile>, usize, usize)>> {
use futures::StreamExt;
let mut in_progress_file = self.create_in_progress_file(request_description)?;
let result = async {
let mut max_record_batch_size = 0;
+ let mut max_batch_rows = 0;
while let Some(batch) = stream.next().await {
let batch = batch?;
let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;
max_record_batch_size = max_record_batch_size.max(gc_sliced_size);
+ max_batch_rows = max_batch_rows.max(batch.num_rows());
}
let file = in_progress_file.finish_async().await?;
- Ok(file.map(|f| (f, max_record_batch_size)))
+ Ok(file.map(|f| (f, max_record_batch_size, max_batch_rows)))
}
.await;
@@ -449,7 +452,7 @@ mod tests {
Box::pin(RecordBatchStreamAdapter::new(schema, input));
let error = manager
- .spill_record_batch_stream_and_return_max_batch_memory(
+ .spill_record_batch_stream_and_return_max_batch_stats(
&mut stream,
"abort-test",
)