Skip to content

Apache DataFusion SQL Query Engine

fix: spill final hash aggregate output when the pool refuses it (#26734)

Yash Aggarwal22eb8ff10238
diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rs
index eec7d0d79f..ddd6162a21 100644
--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs
@@ -619,14 +619,17 @@ impl FinalHashAggregateStream {
.await?;
self.close_input();
- match spill_context.filter(|s| s.has_spills()) {
+ match spill_context {
// - If spilled before, perform merging spill runs
- Some(spill_context) => {
+ Some(spill_context) if spill_context.has_spills() => {
self.produce_output_from_spills(hash_table, spill_context, emitter)
.await?
}
// Either all the input fit in memory or hit soft group limit with no spilling
- None => self.produce_output_from_memory(hash_table, emitter).await?,
+ spill_context => {
+ self.produce_output_from_memory(hash_table, spill_context, emitter)
+ .await?
+ }
}
Ok(())
@@ -806,6 +809,7 @@ impl FinalHashAggregateStream {
async fn produce_output_from_memory(
&mut self,
mut hash_table: AggregateHashTable<FinalMarker>,
+ spill_context: Option<Box<AggregateSpill>>,
mut emitter: TryEmitter<RecordBatch, DataFusionError>,
) -> Result<()> {
let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
@@ -825,7 +829,33 @@ impl FinalHashAggregateStream {
// The table hands over its groups as they are materialized and
// reports a size of 0 once it reaches `Done`, so this releases the
// reservation before the final batch goes downstream.
- self.reservation.try_resize(hash_table.memory_size())?;
+ match self.reservation.try_resize(hash_table.memory_size()) {
+ Ok(()) => {}
+ // The materialized output is bigger than the table was. Write
+ // it to disk and stream it back instead of failing the query.
+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {
+ let spill_context = spill_context.ok_or_else(|| e.context(
+ "Final hash aggregate cannot spill its output because temporary files are not enabled in the DiskManager",
+ ))?;
+ let remaining =
+ std::iter::from_fn(|| hash_table.next_output_batch().transpose());
+ let mut output = spill_context.spill_output(
+ Arc::clone(&self.schema),
+ std::iter::once(Ok(batch)).chain(remaining),
+ )?;
+ drop(hash_table);
+ self.reservation.try_resize(0)?;
+ timer.done();
+
+ while let Some(batch) = output.next().await.transpose()? {
+ emitter
+ .emit(batch.record_output(&self.baseline_metrics))
+ .await;
+ }
+ return Ok(());
+ }
+ Err(e) => return Err(e),
+ }
timer.done();
emitter
diff --git a/datafusion/physical-plan/src/aggregates/spill.rs b/datafusion/physical-plan/src/aggregates/spill.rs
index 5e301cbad7..9da8da2eae 100644
--- a/datafusion/physical-plan/src/aggregates/spill.rs
+++ b/datafusion/physical-plan/src/aggregates/spill.rs
@@ -236,6 +236,27 @@ impl AggregateSpill {
Ok(())
}
+ /// Writes already evaluated output batches to one spill file and returns a
+ /// stream that reads them back.
+ pub(super) fn spill_output(
+ &self,
+ schema: SchemaRef,
+ batches: impl Iterator<Item = Result<RecordBatch>>,
+ ) -> Result<SendableRecordBatchStream> {
+ let spill_manager = SpillManager::new(
+ self.context.runtime_env(),
+ self.spill_manager.metrics.clone(),
+ schema,
+ )
+ .with_compression_type(self.spill_manager.compression);
+ let spill_file = spill_manager
+ .spill_record_batch_iter_and_return_max_batch_memory(batches, self.label)?;
+ let Some((file, max_record_batch_memory)) = spill_file else {
+ return internal_err!("{}: produced an empty output spill", self.label);
+ };
+ spill_manager.read_spill_as_stream(file, Some(max_record_batch_memory))
+ }
+
/// Merges every sorted run, and does the aggregate evaluation with
/// [`ClusteredFinalAggregateStream`].
pub(super) fn into_replay_stream(
diff --git a/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs b/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs
index dccbf34225..853cbabf3c 100644
--- a/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs
+++ b/datafusion/physical-plan/tests/cases/aggregate_memory_call_sequence.rs
@@ -167,6 +167,7 @@ struct Run {
clustering: GroupClusteringMode,
metrics: MetricsSet,
output_batch_rows: Vec<usize>,
+ output: Vec<RecordBatch>,
/// The most bytes the aggregate held at once.
peak: usize,
}
@@ -326,7 +327,7 @@ impl Case {
),
Err(e) => format!("Err({})", e.strip_backtrace()),
};
- drop(result);
+ let output = result.unwrap_or_default();
drop(context);
assert_eq!(pool.reserved(), 0, "every reservation is released");
@@ -335,6 +336,7 @@ impl Case {
clustering: aggregate.group_clustering_mode().clone(),
metrics: aggregate.metrics().unwrap(),
output_batch_rows,
+ output,
peak: pool.peak.load(Ordering::Relaxed),
})
}
@@ -578,11 +580,10 @@ async fn without_spilling_returns_refusal() -> Result<()> {
/// The table fits, but growing the reservation to the materialized output
/// (`try_resize(memory_size())` on the first of several output batches) is
-/// refused. The streams disagree: Final returns the error, Single hands the
-/// output off unreserved (`reserved = false`, no further calls), and Partial
-/// and PartialReduce ignore it (and retry on every batch). Final is the only
-/// one that fails the query, although the output is already materialized,
-/// which #26701 reports. This pins each behavior as it is.
+/// refused. The streams disagree: Final spills the output and reads it back
+/// (#26701), Single hands the output off unreserved (`reserved = false`, no
+/// further calls), and Partial and PartialReduce ignore it (and retry on every
+/// batch). This pins each behavior as it is.
///
/// Built-in aggregates materialize at most a little more than their accounted
/// state (`string_agg` does, see #26701), so this uses [`PaddedOutputUdaf`],
@@ -597,7 +598,8 @@ async fn output_materialization_refused() -> Result<()> {
};
let run = padded(AggregateMode::Final).run().await?;
- assert_eq!(run.spill_count(), 0);
+ assert_eq!(run.spill_count(), 1);
+ assert_eq!(run.output_batch_rows, vec![64; 4]);
assert_snapshot!(run.snapshot, @r"
FinalHashAggregateStream[0] register can_spill=true
FinalHashAggregateStream[0] try_grow [bytes]
@@ -605,9 +607,27 @@ async fn output_materialization_refused() -> Result<()> {
FinalHashAggregateStream[0] try_grow [bytes] -> refused
FinalHashAggregateStream[0] shrink [bytes]
FinalHashAggregateStream[0] unregister
- => Err(Resources exhausted: Failed to allocate additional [bytes] for FinalHashAggregateStream[0] with [bytes] already allocated for this reservation - [bytes] remain available for the total memory pool: greedy(used: [bytes], pool_size: [bytes]))
+ => Ok(256 rows in 4 batches)
");
+ let unlimited = Case {
+ memory_limit: usize::MAX,
+ ..padded(AggregateMode::Final)
+ }
+ .run()
+ .await?;
+ assert_eq!(unlimited.spill_count(), 0);
+ assert_eq!(run.output, unlimited.output);
+
+ let run = Case {
+ spill_enabled: false,
+ ..padded(AggregateMode::Final)
+ }
+ .run()
+ .await?;
+ assert_eq!(run.spill_count(), 0);
+ assert!(run.snapshot.contains("cannot spill its output"));
+
let run = padded(AggregateMode::Single).run().await?;
assert_eq!(run.spill_count(), 0);
assert_eq!(run.output_batch_rows, vec![64; 4]);