Apache DataFusion SQL Query Engine
perf: avoid unnecessary aggregate spill rewrites (#25428)
Chao Sunf029e2b1acf0
diff --git a/datafusion/physical-plan/src/aggregates/spill.rs b/datafusion/physical-plan/src/aggregates/spill.rsindex a1d52382de..887be41855 100644--- a/datafusion/physical-plan/src/aggregates/spill.rs+++ b/datafusion/physical-plan/src/aggregates/spill.rs@@ -116,6 +116,8 @@ pub(super) struct AggregateSpill {spill_manager: SpillManager,/// Spill runs waiting to be merged, all sorted by `spill_expr`.spills: Vec<SortedSpillFile>,+ /// Minimum, across original runs, of each run's largest batch row count.+ min_spill_batch_rows: usize,/// Describes this stream's spill requests, and prefixes its internal errors.label: &'static str,}@@ -200,6 +202,7 @@ impl AggregateSpill {spill_expr,spill_manager,spills: vec![],+ min_spill_batch_rows: batch_size,label,})}@@ -219,6 +222,7 @@ impl AggregateSpill {return Ok(());};+ let max_batch_rows = state_batch.num_rows().min(self.batch_size);let sorted_iter = IncrementalSortIterator::new(state_batch,self.spill_expr.clone(),@@ -239,6 +243,7 @@ impl AggregateSpill {file,max_record_batch_memory,});+ self.min_spill_batch_rows = self.min_spill_batch_rows.min(max_batch_rows);Ok(())}@@ -259,6 +264,7 @@ impl AggregateSpill {spill_expr,spill_manager,spills,+ min_spill_batch_rows,label: _,} = self;@@ -276,6 +282,7 @@ impl AggregateSpill {.with_batch_size(batch_size).with_reservation(merge_reservation).with_replay_headroom()+ .with_intermediate_merge_sizing(Some(min_spill_batch_rows)).build()?;let replay = OrderedFinalAggregateStream::new_with_input_and_metrics(&replay_agg,diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge.rs b/datafusion/physical-plan/src/sorts/multi_level_merge.rsindex c1fe893e0d..28a8000265 100644--- a/datafusion/physical-plan/src/sorts/multi_level_merge.rs+++ b/datafusion/physical-plan/src/sorts/multi_level_merge.rs@@ -159,6 +159,8 @@ pub(crate) struct MultiLevelMergeBuilder {merge_pool: Option<Arc<MergeMemoryPool>>,/// Leave memory for the aggregate consuming the merged spill rows.reserve_replay_headroom: bool,+ /// Avoid merging more runs than needed before the final replay pass.+ size_intermediate_merges: bool,fetch: Option<usize>,enable_round_robin_tie_breaker: bool,}@@ -199,6 +201,7 @@ impl MultiLevelMergeBuilder {reservation,merge_pool: None,reserve_replay_headroom: false,+ size_intermediate_merges: false,enable_round_robin_tie_breaker,fetch,}@@ -216,6 +219,17 @@ impl MultiLevelMergeBuilder {self}+ pub(super) fn with_intermediate_merge_sizing(+ mut self,+ min_spill_batch_rows: Option<usize>,+ ) -> Self {+ // Short inputs can produce larger batches when merged, reducing the next+ // pass's fan-in. Fetch can also leave intermediate runs with short batches.+ self.size_intermediate_merges = self.fetch.is_none()+ && min_spill_batch_rows.is_some_and(|rows| rows >= self.batch_size);+ self+ }+pub(crate) fn create_spillable_merge_stream(self) -> SendableRecordBatchStream {Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&self.schema),@@ -275,9 +289,9 @@ impl MultiLevelMergeBuilder {}// Need to sort to a spill file- let Some((spill_file, max_record_batch_memory)) = self+ let Some((spill_file, max_record_batch_memory, max_batch_rows)) = self.spill_manager- .spill_record_batch_stream_and_return_max_batch_memory(+ .spill_record_batch_stream_and_return_max_batch_stats(&mut stream,"MultiLevelMergeBuilder intermediate spill",)@@ -286,6 +300,10 @@ impl MultiLevelMergeBuilder {continue;};+ // Offset overflow can flush partial batches even without a fetch limit.+ // Keep sizing disabled once a run cannot fill its output batch limit.+ self.size_intermediate_merges &= max_batch_rows >= batch_size_limit;+// Add the spill file paired with the batch-size limit of the merge that// produced it: if that merge consumed a shrunk (skew-resolved) run, its// output was capped and this intermediate run is likewise capped, so a@@ -373,7 +391,7 @@ impl MultiLevelMergeBuilder {let minimum_number_of_required_streams =2_usize.saturating_sub(self.sorted_streams.len());- let (sorted_spill_files, buffer_size) = match self+ let (mut sorted_spill_files, buffer_size) = match self.get_sorted_spill_files_to_merge(2,// we must have at least 2 streams to merge@@ -393,6 +411,47 @@ impl MultiLevelMergeBuilder {}};+ if self.size_intermediate_merges+ && self.sorted_streams.is_empty()+ && !self.sorted_spill_files.is_empty()+ {+ // The next admission starts with two read buffers per run.+ // Fit both those buffers and any replay headroom in the+ // existing grant: another partition may take every spare+ // byte while this intermediate pass writes to disk.+ let final_fan_in = sorted_spill_files.len() * buffer_size / 2;+ let final_fan_in = if self.reserve_replay_headroom {+ final_fan_in / 2+ } else {+ final_fan_in+ };+ let needed = self.sorted_spill_files.len() + sorted_spill_files.len()+ - final_fan_in+ + 1;+ if needed < sorted_spill_files.len() {+ let (first, first_limit) = &sorted_spill_files[0];+ // A wider pending run or a smaller output batch limit can+ // change the next pass's fan-in and add intermediate work.+ // Apply this sizing heuristic only to uniform run budgets.+ let uniform_runs = sorted_spill_files+ .iter()+ .chain(&self.sorted_spill_files)+ .all(|(file, limit)| {+ file.max_record_batch_memory+ == first.max_record_batch_memory+ && limit == first_limit+ });+ if uniform_runs {+ let unused = sorted_spill_files.split_off(needed);+ self.sorted_spill_files.splice(0..0, unused);+ // Keep the grant through intermediate EOF and the+ // spill write, then transfer it to the next admission.+ // This does not reserve additional pool memory.+ self.reservation = memory_reservation.take();+ }+ }+ }+// Don't account for existing streams memory// as we are not holding the memory for themlet mut sorted_streams = mem::take(&mut self.sorted_streams);@@ -751,7 +810,7 @@ impl MultiLevelMergeBuilder {let result = self.spill_manager- .spill_record_batch_stream_and_return_max_batch_memory(+ .spill_record_batch_stream_and_return_max_batch_stats(&mut halved,"MultiLevelMergeBuilder split skewed spill",)@@ -759,7 +818,7 @@ impl MultiLevelMergeBuilder {reservation.free();- let Some((file, new_max)) = result else {+ let Some((file, new_max, new_max_batch_rows)) = result else {return internal_err!("re-spilling a skewed spill file produced no data");};@@ -787,6 +846,7 @@ impl MultiLevelMergeBuilder {// larger output batch formed by concatenating those rows.1};+ self.size_intermediate_merges &= new_max_batch_rows >= new_batch_size_limit;// Push the re-spilled (smaller) file and swap it back into `index`, undoing// the swap-to-back above so the order is preserved.@@ -906,6 +966,10 @@ impl RecordBatchStream for StreamAttachedReservation {}}+#[cfg(test)]+#[path = "multi_level_merge/intermediate_sizing_tests.rs"]+mod intermediate_sizing_tests;+#[cfg(test)]mod tests {use super::*;@@ -923,11 +987,14 @@ mod tests {ExecutionPlanMetricsSet, SpillMetrics,};- fn test_schema() -> SchemaRef {+ pub(super) fn test_schema() -> SchemaRef {Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)]))}- fn build_spill_manager(env: &Arc<RuntimeEnv>, schema: &SchemaRef) -> SpillManager {+ pub(super) fn build_spill_manager(+ env: &Arc<RuntimeEnv>,+ schema: &SchemaRef,+ ) -> SpillManager {SpillManager::new(Arc::clone(env),SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0),@@ -937,7 +1004,7 @@ mod tests {/// Spill `values` (which must already be sorted) as a single sorted run and/// return it as a `SortedSpillFile` carrying its recorded largest-batch memory.- fn make_sorted_spill_file(+ pub(super) fn make_sorted_spill_file(spill_manager: &SpillManager,schema: &SchemaRef,values: Vec<i64>,@@ -961,7 +1028,7 @@ mod tests {}}- fn build_merge_builder(+ pub(super) fn build_merge_builder(spill_manager: SpillManager,schema: SchemaRef,sorted_spill_files: Vec<SortedSpillFile>,@@ -985,6 +1052,150 @@ mod tests {)}+ #[rstest::rstest]+ #[case::full_batches(8, 128, 6)]+ #[case::replay_headroom_limits_sizing(9, 128, 7)]+ #[case::unchanged_selection(13, 128, 7)]+ #[case::short_batches(9, 64, 7)]+ #[tokio::test]+ async fn intermediate_merge_sizing_preserves_final_output(+ #[values(false, true)] size_intermediate_merges: bool,+ #[case] runs: usize,+ #[case] rows: usize,+ #[case] sized_intermediate_inputs: usize,+ ) -> Result<()> {+ let env = Arc::new(RuntimeEnv::default());+ let schema = test_schema();+ let spill_manager = build_spill_manager(&env, &schema);+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();+ let files: Vec<_> = (0..runs)+ .map(|run| {+ make_sorted_spill_file(+ &spill_manager,+ &schema,+ (0..rows).map(|row| (row * runs + run) as i64).collect(),+ )+ })+ .collect();+ let batch_memory = files[0].max_record_batch_memory;+ // Ordinary admission seats seven inputs with read-ahead and equal replay+ // headroom. The retained buffer grant can admit three final inputs plus+ // their replay headroom, so eight runs need six merged inputs. Nine or+ // thirteen runs require all seven and exercise the unchanged selection.+ // Short input batches can grow when merged, so they must keep the original+ // selection to avoid adding a pass with the larger output batch budget.+ let pool: Arc<dyn MemoryPool> =+ Arc::new(GreedyMemoryPool::new(8 * 7 * batch_memory));+ let mut builder =+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, 128)+ .with_replay_headroom(true);+ if size_intermediate_merges {+ builder = builder.with_intermediate_merge_sizing(Some(rows));+ }+ let stream = builder.create_spillable_merge_stream();+ let batches: Vec<RecordBatch> = stream.try_collect().await?;+ let merged = concat_batches(&schema, &batches)?;+ let expected = Int64Array::from_iter_values(0..(runs * rows) as i64);+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);+ let intermediate_inputs = if size_intermediate_merges {+ sized_intermediate_inputs+ } else {+ 7+ };+ assert_eq!(spilled_rows.value(), (runs + intermediate_inputs) * rows);+ assert_eq!(pool.reserved(), 0);+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);+ assert_eq!(env.disk_manager.used_disk_space(), 0);+ Ok(())+ }++ #[rstest::rstest]+ #[tokio::test]+ async fn intermediate_merge_sizing_preserves_heterogeneous_runs(+ #[values(false, true)] size_intermediate_merges: bool,+ ) -> Result<()> {+ use arrow::array::StringArray;+ use arrow::buffer::{Buffer, OffsetBuffer};++ let env = Arc::new(RuntimeEnv::default());+ let schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Int64, false),+ Field::new("payload", DataType::Utf8, false),+ ]));+ let spill_manager = build_spill_manager(&env, &schema);+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();+ let rows = 128;+ let mut files = Vec::new();+ for run in 0..8 {+ let width = if run == 6 { 256 } else { 16 };+ let payload = StringArray::new(+ OffsetBuffer::new(+ (0..=rows)+ .map(|row| (row * width) as i32)+ .collect::<Vec<_>>()+ .into(),+ ),+ Buffer::from(vec![b'x'; rows * width]),+ None,+ );+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from_iter_values(+ (run * rows..(run + 1) * rows).map(|value| value as i64),+ )),+ Arc::new(payload),+ ],+ )?;+ let (file, max_record_batch_memory) = spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(+ std::iter::once(Ok(batch)),+ "heterogeneous input run",+ )?+ .unwrap();+ files.push(SortedSpillFile {+ file,+ max_record_batch_memory,+ });+ }+ let small = files[0].max_record_batch_memory;+ let wide = files[6].max_record_batch_memory;+ // Admit six small inputs first, then the wide input and one small input.+ // Trimming the first merge to three wrongly assumes its fan-in will also+ // fit the wider suffix, adding a later rewrite of the small runs.+ let pool: Arc<dyn MemoryPool> =+ Arc::new(GreedyMemoryPool::new(8 * (wide + small + small / 2)));+ let builder =+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, rows)+ .with_replay_headroom(true)+ .with_intermediate_merge_sizing(size_intermediate_merges.then_some(rows));+ let batches: Vec<RecordBatch> = builder+ .create_spillable_merge_stream()+ .try_collect()+ .await?;+ let merged = concat_batches(&schema, &batches)?;+ let expected = Int64Array::from_iter_values(0..(8 * rows) as i64);+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);+ let small_payload = "x".repeat(16);+ let wide_payload = "x".repeat(256);+ let expected_payload = StringArray::from_iter_values((0..8).flat_map(|run| {+ std::iter::repeat_n(+ if run == 6 {+ &wide_payload+ } else {+ &small_payload+ },+ rows,+ )+ }));+ assert_eq!(merged.column(1).as_string::<i32>(), &expected_payload);+ assert_eq!(spilled_rows.value(), 16 * rows);+ assert_eq!(pool.reserved(), 0);+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);+ assert_eq!(env.disk_manager.used_disk_space(), 0);+ Ok(())+ }+/// Two sorted runs whose largest batches are too big to both/// be seated in the merge budget at once are re-spilled (halved) until they/// fit, and the merge then completes with fully sorted, complete output.diff --git a/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rs b/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rsnew file mode 100644index 0000000000..4841d4b59f--- /dev/null+++ b/datafusion/physical-plan/src/sorts/multi_level_merge/intermediate_sizing_tests.rs@@ -0,0 +1,251 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use super::tests::{+ build_merge_builder, build_spill_manager, make_sorted_spill_file, test_schema,+};+use super::*;+use arrow::array::{AsArray, Int64Array};+use arrow::compute::concat_batches;+use arrow::datatypes::Int64Type;+use datafusion_execution::memory_pool::{+ GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool,+};+use datafusion_execution::runtime_env::RuntimeEnv;+use std::fmt::Display;+use std::sync::Mutex;++#[derive(Debug, Clone, Copy)]+enum PressurePoint {+ ReplayHeadroom,+ IntermediateEof,+}++/// Schedule another consumer immediately when the merge returns memory to the+/// shared pool. Taking only available bytes models an ordinary competing+/// reservation while making both release boundaries deterministic.+#[derive(Debug)]+struct CompetingMemoryPool {+ inner: Arc<dyn MemoryPool>,+ contender: MemoryReservation,+ capacity: usize,+ contender_target: usize,+ pressure_point: PressurePoint,+ active: Mutex<bool>,+}++impl CompetingMemoryPool {+ fn new(+ capacity: usize,+ contender_target: usize,+ pressure_point: PressurePoint,+ ) -> Self {+ let inner: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(capacity));+ let contender = MemoryConsumer::new("competing partition").register(&inner);+ Self {+ inner,+ contender,+ capacity,+ contender_target,+ pressure_point,+ active: Mutex::new(false),+ }+ }+}++impl Display for CompetingMemoryPool {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "CompetingMemoryPool({})", self.inner)+ }+}++impl MemoryPool for CompetingMemoryPool {+ fn name(&self) -> &str {+ "competing"+ }++ fn grow(&self, reservation: &MemoryReservation, additional: usize) {+ self.inner.grow(reservation, additional);+ }++ fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {+ self.inner.shrink(reservation, shrink);+ if shrink == 0 {+ return;+ }+ let mut active = self.active.lock().unwrap();+ *active |= match self.pressure_point {+ PressurePoint::ReplayHeadroom => reservation.size() > 0,+ PressurePoint::IntermediateEof => reservation.size() == 0,+ };+ if *active {+ let additional = (self.contender_target - self.contender.size())+ .min(self.capacity - self.inner.reserved());+ self.contender.try_grow(additional).unwrap();+ }+ }++ fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {+ self.inner.try_grow(reservation, additional)+ }++ fn reserved(&self) -> usize {+ self.inner.reserved()+ }++ fn memory_limit(&self) -> MemoryLimit {+ self.inner.memory_limit()+ }+}++#[rstest::rstest]+#[case::headroom_release(PressurePoint::ReplayHeadroom)]+#[case::intermediate_eof(PressurePoint::IntermediateEof)]+#[tokio::test]+async fn intermediate_sizing_does_not_add_rewrites_under_contention(+ #[case] pressure_point: PressurePoint,+ #[values(2, 3)] final_fan_in: usize,+ #[values(8, 9)] runs: usize,+) -> Result<()> {+ const ROWS: usize = 128;+ let mut rewrites = Vec::new();+ for size_intermediate_merges in [false, true] {+ let env = Arc::new(RuntimeEnv::default());+ let schema = test_schema();+ let spill_manager = build_spill_manager(&env, &schema);+ let spilled_rows = spill_manager.metrics.spilled_rows.clone();+ let files: Vec<_> = (0..runs)+ .map(|run| {+ make_sorted_spill_file(+ &spill_manager,+ &schema,+ (0..ROWS).map(|row| (row * runs + run) as i64).collect(),+ )+ })+ .collect();+ let batch_memory = files[0].max_record_batch_memory;+ // Seven runs initially fit with read-ahead and equal replay headroom.+ // The competing partition takes released bytes until only two or three+ // runs would fit a fresh admission. Both runs use the same release hook.+ let competing_pool = Arc::new(CompetingMemoryPool::new(+ 8 * 7 * batch_memory,+ 8 * (7 - final_fan_in) * batch_memory,+ pressure_point,+ ));+ let pool = Arc::clone(&competing_pool) as Arc<dyn MemoryPool>;+ let builder =+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, ROWS)+ .with_replay_headroom(true)+ .with_intermediate_merge_sizing(size_intermediate_merges.then_some(ROWS));+ let mut stream = builder.create_spillable_merge_stream();+ let first = stream.try_next().await?.expect("nonempty merge");+ if matches!(pressure_point, PressurePoint::IntermediateEof)+ && size_intermediate_merges+ && runs == 8+ {+ // Sizing these eight runs retains the admitted buffers across the+ // intermediate EOF. The contender cannot take them before the final+ // merge is admitted, so this boundary must not have fired yet.+ assert_eq!(competing_pool.contender.size(), 0);+ } else {+ assert_eq!(+ competing_pool.contender.size(),+ competing_pool.contender_target+ );+ }+ let mut batches = vec![first];+ batches.extend(stream.try_collect::<Vec<_>>().await?);+ let merged = concat_batches(&schema, &batches)?;+ let expected = Int64Array::from_iter_values(0..(runs * ROWS) as i64);+ assert_eq!(merged.column(0).as_primitive::<Int64Type>(), &expected);+ rewrites.push(spilled_rows.value() - runs * ROWS);+ assert_eq!(+ competing_pool.contender.size(),+ competing_pool.contender_target+ );+ assert_eq!(pool.reserved(), competing_pool.contender.size());+ competing_pool.contender.free();+ assert_eq!(pool.reserved(), 0);+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);+ assert_eq!(env.disk_manager.used_disk_space(), 0);+ }+ assert_eq!(+ rewrites[0],+ if runs == 9 && final_fan_in == 2 { 9 } else { 7 } * ROWS+ );+ assert!(+ rewrites[1] <= rewrites[0],+ "sizing rewrote {} rows versus {} without sizing under {pressure_point:?}",+ rewrites[1],+ rewrites[0],+ );+ if runs == 8 {+ assert_eq!(rewrites[1], 6 * ROWS);+ }+ Ok(())+}++#[tokio::test]+async fn sized_merge_retains_reservation_until_builder_drop() -> Result<()> {+ const RUNS: usize = 8;+ const ROWS: usize = 128;+ let env = Arc::new(RuntimeEnv::default());+ let schema = test_schema();+ let spill_manager = build_spill_manager(&env, &schema);+ let files: Vec<_> = (0..RUNS)+ .map(|run| {+ make_sorted_spill_file(+ &spill_manager,+ &schema,+ (0..ROWS).map(|row| (row * RUNS + run) as i64).collect(),+ )+ })+ .collect();+ let batch_memory = files[0].max_record_batch_memory;+ let competing_pool = Arc::new(CompetingMemoryPool::new(+ 8 * 7 * batch_memory,+ 8 * 5 * batch_memory,+ PressurePoint::ReplayHeadroom,+ ));+ let pool = Arc::clone(&competing_pool) as Arc<dyn MemoryPool>;+ let mut builder =+ build_merge_builder(spill_manager, Arc::clone(&schema), files, &pool, ROWS)+ .with_replay_headroom(true)+ .with_intermediate_merge_sizing(Some(ROWS));+ let MergeStep::Stream { stream, .. } =+ builder.merge_sorted_runs_within_mem_limit(false)?+ else {+ panic!("six inputs should be selected for the intermediate merge");+ };+ assert_eq!(builder.sorted_spill_files.len(), 2);+ let retained = 4 * 7 * batch_memory;+ assert_eq!(builder.reservation.size(), retained);+ assert_eq!(pool.reserved(), retained + competing_pool.contender.size());++ // The intermediate stream may finish or be cancelled before the builder+ // starts final admission. Its drop must leave the grant with the builder.+ drop(stream);+ assert_eq!(builder.reservation.size(), retained);+ assert_eq!(pool.reserved(), retained + competing_pool.contender.size());+ drop(builder);+ assert_eq!(pool.reserved(), competing_pool.contender.size());+ competing_pool.contender.free();+ assert_eq!(pool.reserved(), 0);+ assert_eq!(env.disk_manager.spilling_progress().active_files_count, 0);+ assert_eq!(env.disk_manager.used_disk_space(), 0);+ Ok(())+}diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rsindex 61bb7742ee..199dceac8c 100644--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs@@ -98,6 +98,7 @@ pub struct StreamingMergeBuilder<'a> {merge_pool: Option<Arc<MergeMemoryPool>>,/// Leave memory for the aggregate consuming the merged spill rows.reserve_replay_headroom: bool,+ min_spill_batch_rows: Option<usize>,enable_round_robin_tie_breaker: bool,}@@ -170,6 +171,19 @@ impl<'a> StreamingMergeBuilder<'a> {self}+ /// Limit intermediate rewrites when the retained buffer reservation can+ /// also admit the final replay pass with its headroom.+ /// `min_spill_batch_rows` is the minimum largest-batch row count across input+ /// runs. A run without a full batch disables sizing for the whole replay,+ /// since merging short batches can increase the intermediate run's budget.+ pub(crate) fn with_intermediate_merge_sizing(+ mut self,+ min_spill_batch_rows: Option<usize>,+ ) -> Self {+ self.min_spill_batch_rows = min_spill_batch_rows;+ self+ }+/// See [SortPreservingMergeExec::with_round_robin_repartition] for more/// information.///@@ -204,6 +218,7 @@ impl<'a> StreamingMergeBuilder<'a> {reservation,merge_pool,reserve_replay_headroom,+ min_spill_batch_rows,fetch,expressions,enable_round_robin_tie_breaker,@@ -246,6 +261,7 @@ impl<'a> StreamingMergeBuilder<'a> {).with_merge_pool(merge_pool).with_replay_headroom(reserve_replay_headroom)+ .with_intermediate_merge_sizing(min_spill_batch_rows).create_spillable_merge_stream());}diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex 389fafb985..9c07602fab 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -140,29 +140,32 @@ impl SpillManager {Ok(file.map(|f| (f, max_record_batch_size)))}- /// Spill a stream of `RecordBatch`es to disk and return the spill file and the size of the largest batch in memory- pub(crate) async fn spill_record_batch_stream_and_return_max_batch_memory(+ /// Spill a stream of `RecordBatch`es and return the file, maximum batch memory,+ /// and maximum batch row count.+ pub(crate) async fn spill_record_batch_stream_and_return_max_batch_stats(&self,stream: &mut SendableRecordBatchStream,request_description: &str,- ) -> Result<Option<(Arc<dyn SpillFile>, usize)>> {+ ) -> Result<Option<(Arc<dyn SpillFile>, usize, usize)>> {use futures::StreamExt;let mut in_progress_file = self.create_in_progress_file(request_description)?;let result = async {let mut max_record_batch_size = 0;+ let mut max_batch_rows = 0;while let Some(batch) = stream.next().await {let batch = batch?;let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;max_record_batch_size = max_record_batch_size.max(gc_sliced_size);+ max_batch_rows = max_batch_rows.max(batch.num_rows());}let file = in_progress_file.finish_async().await?;- Ok(file.map(|f| (f, max_record_batch_size)))+ Ok(file.map(|f| (f, max_record_batch_size, max_batch_rows)))}.await;@@ -449,7 +452,7 @@ mod tests {Box::pin(RecordBatchStreamAdapter::new(schema, input));let error = manager- .spill_record_batch_stream_and_return_max_batch_memory(+ .spill_record_batch_stream_and_return_max_batch_stats(&mut stream,"abort-test",)