Apache DataFusion SQL Query Engine
fix: spill final hash aggregate output when the pool refuses it (#26734)
Yash Aggarwal22eb8ff10238
diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rsindex eec7d0d79f..ddd6162a21 100644--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs@@ -619,14 +619,17 @@ impl FinalHashAggregateStream {.await?;self.close_input();- match spill_context.filter(|s| s.has_spills()) {+ match spill_context {// - If spilled before, perform merging spill runs- Some(spill_context) => {+ Some(spill_context) if spill_context.has_spills() => {self.produce_output_from_spills(hash_table, spill_context, emitter).await?}// Either all the input fit in memory or hit soft group limit with no spilling- None => self.produce_output_from_memory(hash_table, emitter).await?,+ spill_context => {+ self.produce_output_from_memory(hash_table, spill_context, emitter)+ .await?+ }}Ok(())@@ -806,6 +809,7 @@ impl FinalHashAggregateStream {async fn produce_output_from_memory(&mut self,mut hash_table: AggregateHashTable<FinalMarker>,+ spill_context: Option<Box<AggregateSpill>>,mut emitter: TryEmitter<RecordBatch, DataFusionError>,) -> Result<()> {let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();@@ -825,7 +829,33 @@ impl FinalHashAggregateStream {// The table hands over its groups as they are materialized and// reports a size of 0 once it reaches `Done`, so this releases the// reservation before the final batch goes downstream.- self.reservation.try_resize(hash_table.memory_size())?;+ match self.reservation.try_resize(hash_table.memory_size()) {+ Ok(()) => {}+ // The materialized output is bigger than the table was. Write+ // it to disk and stream it back instead of failing the query.+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {+ let spill_context = spill_context.ok_or_else(|| e.context(+ "Final hash aggregate cannot spill its output because temporary files are not enabled in the DiskManager",+ ))?;+ let remaining =+ std::iter::from_fn(|| hash_table.next_output_batch().transpose());+ let mut output = spill_context.spill_output(+ Arc::clone(&self.schema),+ std::iter::once(Ok(batch)).chain(remaining),+ )?;+ drop(hash_table);+ self.reservation.try_resize(0)?;+ timer.done();++ while let Some(batch) = output.next().await.transpose()? {+ emitter+ .emit(batch.record_output(&self.baseline_metrics))+ .await;+ }+ return Ok(());+ }+ Err(e) => return Err(e),+ }timer.done();emitterdiff --git a/datafusion/physical-plan/src/aggregates/spill.rs b/datafusion/physical-plan/src/aggregates/spill.rsindex 5e301cbad7..9da8da2eae 100644--- a/datafusion/physical-plan/src/aggregates/spill.rs+++ b/datafusion/physical-plan/src/aggregates/spill.rs@@ -236,6 +236,27 @@ impl AggregateSpill {Ok(())}+ /// Writes already evaluated output batches to one spill file and returns a+ /// stream that reads them back.+ pub(super) fn spill_output(+ &self,+ schema: SchemaRef,+ batches: impl Iterator<Item = Result<RecordBatch>>,+ ) -> Result<SendableRecordBatchStream> {+ let spill_manager = SpillManager::new(+ self.context.runtime_env(),+ self.spill_manager.metrics.clone(),+ schema,+ )+ .with_compression_type(self.spill_manager.compression);+ let spill_file = spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(batches, self.label)?;+ let Some((file, max_record_batch_memory)) = spill_file else {+ return internal_err!("{}: produced an empty output spill", self.label);+ };+ spill_manager.read_spill_as_stream(file, Some(max_record_batch_memory))+ }+/// Merges every sorted run, and does the aggregate evaluation with/// [`ClusteredFinalAggregateStream`].pub(super) fn into_replay_stream(diff --git a/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs b/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rsindex dccbf34225..853cbabf3c 100644--- a/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs+++ b/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs@@ -167,6 +167,7 @@ struct Run {clustering: GroupClusteringMode,metrics: MetricsSet,output_batch_rows: Vec<usize>,+ output: Vec<RecordBatch>,/// The most bytes the aggregate held at once.peak: usize,}@@ -326,7 +327,7 @@ impl Case {),Err(e) => format!("Err({})", e.strip_backtrace()),};- drop(result);+ let output = result.unwrap_or_default();drop(context);assert_eq!(pool.reserved(), 0, "every reservation is released");@@ -335,6 +336,7 @@ impl Case {clustering: aggregate.group_clustering_mode().clone(),metrics: aggregate.metrics().unwrap(),output_batch_rows,+ output,peak: pool.peak.load(Ordering::Relaxed),})}@@ -578,11 +580,10 @@ async fn without_spilling_returns_refusal() -> Result<()> {/// The table fits, but growing the reservation to the materialized output/// (`try_resize(memory_size())` on the first of several output batches) is-/// refused. The streams disagree: Final returns the error, Single hands the-/// output off unreserved (`reserved = false`, no further calls), and Partial-/// and PartialReduce ignore it (and retry on every batch). Final is the only-/// one that fails the query, although the output is already materialized,-/// which #26701 reports. This pins each behavior as it is.+/// refused. The streams disagree: Final spills the output and reads it back+/// (#26701), Single hands the output off unreserved (`reserved = false`, no+/// further calls), and Partial and PartialReduce ignore it (and retry on every+/// batch). This pins each behavior as it is.////// Built-in aggregates materialize at most a little more than their accounted/// state (`string_agg` does, see #26701), so this uses [`PaddedOutputUdaf`],@@ -597,7 +598,8 @@ async fn output_materialization_refused() -> Result<()> {};let run = padded(AggregateMode::Final).run().await?;- assert_eq!(run.spill_count(), 0);+ assert_eq!(run.spill_count(), 1);+ assert_eq!(run.output_batch_rows, vec![64; 4]);assert_snapshot!(run.snapshot, @r"FinalHashAggregateStream[0] register can_spill=trueFinalHashAggregateStream[0] try_grow [bytes]@@ -605,9 +607,27 @@ async fn output_materialization_refused() -> Result<()> {FinalHashAggregateStream[0] try_grow [bytes] -> refusedFinalHashAggregateStream[0] shrink [bytes]FinalHashAggregateStream[0] unregister- => Err(Resources exhausted: Failed to allocate additional [bytes] for FinalHashAggregateStream[0] with [bytes] already allocated for this reservation - [bytes] remain available for the total memory pool: greedy(used: [bytes], pool_size: [bytes]))+ => Ok(256 rows in 4 batches)");+ let unlimited = Case {+ memory_limit: usize::MAX,+ ..padded(AggregateMode::Final)+ }+ .run()+ .await?;+ assert_eq!(unlimited.spill_count(), 0);+ assert_eq!(run.output, unlimited.output);++ let run = Case {+ spill_enabled: false,+ ..padded(AggregateMode::Final)+ }+ .run()+ .await?;+ assert_eq!(run.spill_count(), 0);+ assert!(run.snapshot.contains("cannot spill its output"));+let run = padded(AggregateMode::Single).run().await?;assert_eq!(run.spill_count(), 0);assert_eq!(run.output_batch_rows, vec![64; 4]);