Apache DataFusion SQL Query Engine
refactor(hash-aggr): Let partial-reduce mode aggregation early emit when OOM (#24785)
Yongting You1608ee70faa0
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex f00f1a160e..6da7ee1018 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -187,7 +187,6 @@ use datafusion_common::{assert_eq_or_internal_err, internal_err, not_impl_err,};use datafusion_execution::TaskContext;-use datafusion_execution::memory_pool::MemoryLimit;use datafusion_expr::{Accumulator, Aggregate};use datafusion_physical_expr::aggregate::AggregateFunctionExpr;use datafusion_physical_expr::equivalence::ProjectionMapping;@@ -1308,12 +1307,7 @@ impl AggregateExec {&& self.group_by.is_single()}- fn should_use_partial_reduce_hash_stream(&self, context: &TaskContext) -> bool {- // TODO: implement memory-limited path and remove this limitation- if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {- return false;- }-+ fn should_use_partial_reduce_hash_stream(&self, _context: &TaskContext) -> bool {self.mode == AggregateMode::PartialReduce&& self.limit_options.is_none()&& self.input_order_mode == InputOrderMode::Linear@@ -3241,7 +3235,7 @@ mod tests {use arrow::array::{BooleanArray, DictionaryArray, Float32Array, Float64Array, Int32Array,- Int64Array, StructArray, UInt32Array, UInt64Array,+ Int64Array, NullArray, StructArray, UInt32Array, UInt64Array,};use arrow::compute::{SortOptions, concat_batches};use arrow::datatypes::Int32Type;@@ -4586,6 +4580,15 @@ mod tests {}fn partial_reduce_test_aggregate() -> Result<AggregateExec> {+ partial_reduce_test_aggregate_with_batches(1)+ }++ /// Partial-reduce aggregate over `num_input_batches` identical input batches+ /// of partial states, each reducing to groups `1, 2, 3` with sums+ /// `50, 20, 30`.+ fn partial_reduce_test_aggregate_with_batches(+ num_input_batches: usize,+ ) -> Result<AggregateExec> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, false),Field::new("b", DataType::Float64, false),@@ -4618,7 +4621,7 @@ mod tests {],)?;let partial_reduce_input = TestMemoryExec::try_new_exec(- &[vec![partial_state_batch]],+ &[vec![partial_state_batch; num_input_batches]],Arc::clone(&partial_schema),None,)?;@@ -4654,26 +4657,162 @@ mod tests {Ok(())}- /// Spilling behavior is not implemented for partial-reduce stream yet, so fall- /// back to the existing `GroupedHashAggregateStream`+ /// Partial-reduce hash aggregation emits its accumulated partial states early+ /// under memory pressure instead of failing, and the early-emitted states+ /// still merge into the correct result.#[tokio::test]- async fn partial_reduce_aggregate_with_memory_limit_planning() -> Result<()> {- let partial_reduce = partial_reduce_test_aggregate()?;+ async fn partial_reduce_aggregate_with_memory_limit_emits_early() -> Result<()> {+ let num_input_batches = 3;+ let partial_reduce =+ partial_reduce_test_aggregate_with_batches(num_input_batches)?;let runtime = RuntimeEnvBuilder::new().with_memory_limit(1, 1.0).build_arc()?;- let task_ctx =- Arc::new(- TaskContext::default()- .with_session_config(SessionConfig::new().set_bool(- "datafusion.execution.enable_migration_aggregate",- true,- ))- .with_runtime(runtime),- );+ // A batch size smaller than the number of flushed groups also covers+ // splitting one flush across several output batches.+ let batch_size = 2;+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(migrated_hash_session_config(batch_size))+ .with_runtime(runtime),+ );let stream = partial_reduce.execute_typed(0, &task_ctx)?;- assert!(matches!(stream, StreamType::GroupedHash(_)));+ assert!(matches!(stream, StreamType::PartialReduceHash(_)));+ let stream: SendableRecordBatchStream = stream.into();+ let output = collect(stream).await?;++ // The table is flushed after every input batch, so each of the three+ // groups is emitted once per input batch instead of being merged into a+ // single row. Each flush is sliced into batches of 2 and 1 rows.+ assert_eq!(output.len(), 2 * num_input_batches);+ assert_snapshot!(batches_to_string(&output), @r"+ +---+-------------++ | a | SUM(b)[sum] |+ +---+-------------++ | 1 | 50.0 |+ | 2 | 20.0 |+ | 3 | 30.0 |+ | 1 | 50.0 |+ | 2 | 20.0 |+ | 3 | 30.0 |+ | 1 | 50.0 |+ | 2 | 20.0 |+ | 3 | 30.0 |+ +---+-------------++ ");++ Ok(())+ }++ /// Same shape as [`partial_reduce_test_aggregate_with_batches`], but with multiple+ /// group keys.+ fn partial_reduce_test_aggregate_rows_multi_group_keys(+ num_input_batches: usize,+ ) -> Result<AggregateExec> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::UInt32, false),+ Field::new("n", DataType::Null, true),+ Field::new("b", DataType::Float64, false),+ ]));+ let group_by = PhysicalGroupBy::new_single(vec![+ (col("a", &schema)?, "a".to_string()),+ (col("n", &schema)?, "n".to_string()),+ ]);+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("SUM(b)")+ .build()?,+ )];++ let empty_input =+ TestMemoryExec::try_new_exec(&[vec![]], Arc::clone(&schema), None)?;+ let partial = AggregateExec::try_new(+ AggregateMode::Partial,+ group_by.clone(),+ aggregates.clone(),+ vec![None],+ empty_input,+ Arc::clone(&schema),+ )?;+ let partial_schema = partial.schema();+ let partial_state_batch = RecordBatch::try_new(+ Arc::clone(&partial_schema),+ vec![+ Arc::new(UInt32Array::from(vec![1, 2, 1, 3])),+ Arc::new(NullArray::new(4)),+ Arc::new(Float64Array::from(vec![10.0, 20.0, 40.0, 30.0])),+ ],+ )?;+ let partial_reduce_input = TestMemoryExec::try_new_exec(+ &[vec![partial_state_batch; num_input_batches]],+ Arc::clone(&partial_schema),+ None,+ )?;++ AggregateExec::try_new(+ AggregateMode::PartialReduce,+ group_by,+ aggregates,+ vec![None],+ partial_reduce_input,+ partial_schema,+ )+ }++ #[tokio::test]+ async fn partial_reduce_aggregate_with_memory_limit_emits_early_multi_group_keys()+ -> Result<()> {+ let num_input_batches = 3;+ let partial_reduce =+ partial_reduce_test_aggregate_rows_multi_group_keys(num_input_batches)?;++ // Pin the representation: this is exactly the condition+ // `new_group_values` uses to pick `GroupValuesRows` over+ // `GroupValuesColumn`. If a `Null` `GroupColumn` is ever added, this+ // assertion fires and the test stops covering the row-encoded path.+ let group_schema = partial_reduce+ .group_by+ .group_schema(&partial_reduce.schema())?;+ assert!(+ !group_values::multi_group_by::supported_schema(&group_schema),+ "expected the Null group column to force the GroupValuesRows fallback"+ );++ let runtime = RuntimeEnvBuilder::new()+ .with_memory_limit(1, 1.0)+ .build_arc()?;+ let batch_size = 2;+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(migrated_hash_session_config(batch_size))+ .with_runtime(runtime),+ );++ let stream = partial_reduce.execute_typed(0, &task_ctx)?;+ assert!(matches!(stream, StreamType::PartialReduceHash(_)));+ let stream: SendableRecordBatchStream = stream.into();+ let output = collect(stream).await?;++ // Same flush cadence as the column-backed test: one flush per input+ // batch, each sliced into batches of 2 and 1 rows.+ assert_eq!(output.len(), 2 * num_input_batches);+ assert_snapshot!(batches_to_string(&output), @r"+ +---+---+-------------++ | a | n | SUM(b)[sum] |+ +---+---+-------------++ | 1 | | 50.0 |+ | 2 | | 20.0 |+ | 3 | | 30.0 |+ | 1 | | 50.0 |+ | 2 | | 20.0 |+ | 3 | | 30.0 |+ | 1 | | 50.0 |+ | 2 | | 20.0 |+ | 3 | | 30.0 |+ +---+---+-------------++ ");Ok(())}diff --git a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rsindex 9eedf86847..d8f1447bc9 100644--- a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs+++ b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs@@ -28,7 +28,7 @@ use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::Result;+use datafusion_common::{DataFusionError, Result};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};use futures::stream::{Stream, StreamExt};@@ -68,6 +68,15 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};/// This stage is useful for tree-reduce plans. It consumes the same schema as/// a final aggregate stage, but emits the same schema as a partial aggregate/// stage.+///+/// # Memory Management+///+/// If the memory reservation cannot grow after aggregating an input batch, all+/// accumulated partial states are emitted immediately, and the remaining input+/// is aggregated with an empty table. This repeats until the input ends.+///+/// See [`crate::aggregates::AggregateMode::PartialReduce`] for why it's allowed+/// to emit the same group multiple times.pub(crate) struct PartialReduceHashAggregateStream {/// Output schema: group columns followed by partial aggregate state columns.schema: SchemaRef,@@ -75,6 +84,9 @@ pub(crate) struct PartialReduceHashAggregateStream {/// Input batches containing partial aggregate state rows.input: SendableRecordBatchStream,+ /// Target output batch size from configuration.+ batch_size: usize,+/// Execution metrics shared with the aggregate plan node.baseline_metrics: BaselineMetrics,@@ -93,10 +105,22 @@ enum PartialReduceHashAggregateState {ReadingInput {hash_table: AggregateHashTable<PartialReduceMarker>,},+ /// A fully materialized partial-state batch being emitted incrementally+ /// because the table ran out of memory while reading input.+ EmittingOnMemoryPressure {+ hash_table: AggregateHashTable<PartialReduceMarker>,+ // After each incremental emitting step, `remaining_groups` is updated+ // with batch slicing.+ remaining_groups: RecordBatch,+ },ProducingOutput {hash_table: AggregateHashTable<PartialReduceMarker>,},Done,+ /// Sentinel state to use when returning error from any other states, because:+ /// - It explicitly releases state-owned resources immediately+ /// - More defensive against accidentally resuming execution after error+ Error,}type PartialReduceHashAggregatePoll = Poll<Option<Result<RecordBatch>>>;@@ -111,28 +135,34 @@ type PartialReduceHashAggregateStateTransition = ControlFlow<impl PartialReduceHashAggregateState {fn hash_table(&self) -> &AggregateHashTable<PartialReduceMarker> {match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table+ Self::ReadingInput { hash_table }+ | Self::EmittingOnMemoryPressure { hash_table, .. }+ | Self::ProducingOutput { hash_table } => hash_table,+ Self::Done | Self::Error => {+ unreachable!("Done and Error states do not hold a hash table")}- Self::Done => unreachable!("Done state does not hold a hash table"),}}fn hash_table_mut(&mut self) -> &mut AggregateHashTable<PartialReduceMarker> {match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table+ Self::ReadingInput { hash_table }+ | Self::EmittingOnMemoryPressure { hash_table, .. }+ | Self::ProducingOutput { hash_table } => hash_table,+ Self::Done | Self::Error => {+ unreachable!("Done and Error states do not hold a hash table")}- Self::Done => unreachable!("Done state does not hold a hash table"),}}fn into_hash_table(self) -> AggregateHashTable<PartialReduceMarker> {match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table+ Self::ReadingInput { hash_table }+ | Self::EmittingOnMemoryPressure { hash_table, .. }+ | Self::ProducingOutput { hash_table } => hash_table,+ Self::Done | Self::Error => {+ unreachable!("Done and Error states do not hold a hash table")}- Self::Done => unreachable!("Done state does not hold a hash table"),}}@@ -178,19 +208,25 @@ impl PartialReduceHashAggregateStream {Ok(Self {schema,input,+ batch_size,baseline_metrics,reservation,state: Some(PartialReduceHashAggregateState::ReadingInput { hash_table }),})}- fn start_output(- &mut self,- hash_table: &mut AggregateHashTable<PartialReduceMarker>,- ) -> Result<()> {+ fn close_input(&mut self) {let input_schema = self.input.schema();self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));- hash_table.start_output()+ }++ fn break_with_err(+ error: DataFusionError,+ ) -> PartialReduceHashAggregateStateTransition {+ ControlFlow::Break((+ Poll::Ready(Some(Err(error))),+ PartialReduceHashAggregateState::Error,+ ))}/// Handle ReadingInput state - aggregate partial state batches into the hash table.@@ -219,46 +255,126 @@ impl PartialReduceHashAggregateStream {timer.done();if let Err(e) = result {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- original_state,- ));+ return Self::break_with_err(e);}- if let Err(e) = self- .reservation- .try_resize(original_state.hash_table().memory_size())- {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- original_state,- ));- }-- ControlFlow::Continue(original_state)- }- Poll::Ready(Some(Err(e))) => {- ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state))+ // Update the memory reservation. If OOM, do early emit.+ self.resize_or_emit_early(original_state)}+ Poll::Ready(Some(Err(e))) => Self::break_with_err(e),// Input ends, move to output statePoll::Ready(None) => {+ self.close_input();let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();let timer = elapsed_compute.timer();- let result = self.start_output(original_state.hash_table_mut());+ let result = original_state.hash_table_mut().start_output();timer.done();match result {Ok(()) => {ControlFlow::Continue(original_state.into_producing_output())}- Err(e) => {- ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state))- }+ Err(e) => Self::break_with_err(e),}}}}+ /// Update the memory reservation. If the reservation succeeds, continue reading+ /// input. If OOM, clear the aggregated states in the hash table, and early emit+ /// them immediately.+ ///+ /// Returns the next state; the caller finishes the intended task based on it.+ ///+ /// The reservation is left at its pre-emission size while the states are being+ /// emitted, because the cleared states are still held in memory as+ /// `remaining_groups`. The reservation will be reset after exiting the+ /// `EmittingOnMemoryPressure` state.+ ///+ /// # Implementation Note+ /// All accumulated states are materialized at once, and then sliced into+ /// `batch_size` output batches. Emit them incrementally after blocked state+ /// management is ready.+ ///+ /// Issue: <https://github.com/apache/datafusion/issues/7065>+ fn resize_or_emit_early(+ &mut self,+ mut original_state: PartialReduceHashAggregateState,+ ) -> PartialReduceHashAggregateStateTransition {+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let _timer = elapsed_compute.timer(); // Stop on drop+ let resize_result = self+ .reservation+ .try_resize(original_state.hash_table().memory_size());++ let oom = match resize_result {+ Ok(()) => return ControlFlow::Continue(original_state),+ Err(e @ DataFusionError::ResourcesExhausted(_)) => e,+ Err(e) => return Self::break_with_err(e),+ };++ let state_batch_result = original_state.hash_table_mut().take_state_batch();++ match state_batch_result {+ Ok(Some(remaining_groups)) => ControlFlow::Continue(+ PartialReduceHashAggregateState::EmittingOnMemoryPressure {+ hash_table: original_state.into_hash_table(),+ remaining_groups,+ },+ ),+ // No accumulated group to emit, so early emission cannot release any+ // memory: report the original error.+ Ok(None) => Self::break_with_err(oom),+ Err(e) => Self::break_with_err(e),+ }+ }++ /// Handle EmittingOnMemoryPressure state - emit a materialized partial-state+ /// batch in `batch_size`(from configuration) slices. After all slices are+ /// emitted, update the memory reservation and resume reading input.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_emitting_on_memory_pressure(+ &mut self,+ original_state: PartialReduceHashAggregateState,+ ) -> PartialReduceHashAggregateStateTransition {+ let PartialReduceHashAggregateState::EmittingOnMemoryPressure {+ hash_table,+ remaining_groups: batch,+ } = original_state+ else {+ unreachable!("expected the EmittingOnMemoryPressure state")+ };++ let (output_batch, next_state) = if batch.num_rows() <= self.batch_size {+ // Go back to `ReadingInput`+ (+ batch,+ PartialReduceHashAggregateState::ReadingInput { hash_table },+ )+ } else {+ // More batches to output, continue in the current state.+ let remaining =+ batch.slice(self.batch_size, batch.num_rows() - self.batch_size);+ let output = batch.slice(0, self.batch_size);+ (+ output,+ PartialReduceHashAggregateState::EmittingOnMemoryPressure {+ hash_table,+ remaining_groups: remaining,+ },+ )+ };++ debug_assert!(output_batch.num_rows() > 0);+ ControlFlow::Break((+ Poll::Ready(Some(Ok(output_batch.record_output(&self.baseline_metrics)))),+ next_state,+ ))+ }+/// Handle ProducingOutput state - emit merged partial aggregate state batches.////// See comments at `poll_next()` for details.@@ -281,6 +397,8 @@ impl PartialReduceHashAggregateStream {match result {Ok(Some(batch)) => {+ // The output is already materialized, so a failed resize cannot+ // be acted on: keep the reservation as is and finish the output.let _ = self.reservation.try_resize(original_state.hash_table().memory_size());@@ -300,7 +418,7 @@ impl PartialReduceHashAggregateStream {let _ = self.reservation.try_resize(0);ControlFlow::Continue(original_state.into_done())}- Err(e) => ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state)),+ Err(e) => Self::break_with_err(e),}}}@@ -325,10 +443,22 @@ impl Stream for PartialReduceHashAggregateStream {/// Aggregate one partial-state input batch, update the inner aggregate/// hash table, and continue with the next input batch.///+ /// -> EmittingOnMemoryPressure+ /// The table cannot reserve enough memory. Materialize all accumulated+ /// partial states and begin emitting them incrementally.+ ////// -> ProducingOutput/// Input was exhausted. Move to the next state to start outputting/// merged partial aggregate states.///+ /// EmittingOnMemoryPressure+ /// -> EmittingOnMemoryPressure+ /// One batch-sized slice was yielded; repeat until all materialized+ /// partial states are emitted.+ ///+ /// -> ReadingInput+ /// The materialized states were emitted; continue with the empty table.+ ////// ProducingOutput/// -> ProducingOutput/// One merged partial-state output batch was yielded; repeat to@@ -337,6 +467,13 @@ impl Stream for PartialReduceHashAggregateStream {/// -> Done/// All merged partial-state output was emitted.///+ /// Any active state+ /// -> Error+ /// An error drops state-owned resources before it is returned.+ ///+ /// Error+ /// -> (end)+ ////// Done/// -> (end)/// ```@@ -354,9 +491,18 @@ impl Stream for PartialReduceHashAggregateStream {state @ PartialReduceHashAggregateState::ReadingInput { .. } => {self.handle_reading_input(cx, state)}+ state @ PartialReduceHashAggregateState::EmittingOnMemoryPressure {+ ..+ } => self.handle_emitting_on_memory_pressure(state),state @ PartialReduceHashAggregateState::ProducingOutput { .. } => {self.handle_producing_output(state)}+ state @ PartialReduceHashAggregateState::Error => {+ self.close_input();+ self.reservation.free();+ self.state = Some(state);+ return Poll::Ready(None);+ }state @ PartialReduceHashAggregateState::Done => {let _ = self.reservation.try_resize(0);self.state = Some(state);@@ -368,6 +514,19 @@ impl Stream for PartialReduceHashAggregateStream {ControlFlow::Continue(next_state) => {self.state = Some(next_state);}+ ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {+ debug_assert!(matches!(+ next_state,+ PartialReduceHashAggregateState::Error+ ));++ // The handler has already discarded its state-owned resources.+ // Release the remaining stream-owned resources before returning.+ self.close_input();+ self.reservation.free();+ self.state = Some(PartialReduceHashAggregateState::Error);+ return Poll::Ready(Some(Err(e)));+ }ControlFlow::Break((poll, next_state)) => {self.state = Some(next_state);return poll;