Apache DataFusion SQL Query Engine
chore(`PartialReduceHashAggregateStream`): convert to async generators and cleanup (#24015)
Raz Luvaton6bbd73aa188e
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex b25caf815e..020196e7da 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -410,10 +410,6 @@ impl<AggrMode> AggregateHashTable<AggrMode> {matches!(self.state, AggregateHashTableState::Building(_))}- pub(in crate::aggregates) fn is_done(&self) -> bool {- matches!(self.state, AggregateHashTableState::Done)- }-pub(super) fn start_outputting(&mut self) {let AggregateHashTableState::Building(mut state) =std::mem::replace(&mut self.state, AggregateHashTableState::Done)diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex e528cf467a..302da07f33 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -716,7 +716,7 @@ impl From<StreamType> for SendableRecordBatchStream {match stream {StreamType::AggregateStream(stream) => Box::pin(stream),StreamType::PartialHash(stream) => stream.into_stream(),- StreamType::PartialReduceHash(stream) => Box::pin(stream),+ StreamType::PartialReduceHash(stream) => stream.into_stream(),StreamType::FinalHash(stream) => stream.into_stream(),StreamType::SingleHash(stream) => stream.into_stream(),StreamType::OrderedPartialAggregate(stream) => stream.into_stream(),@@ -6069,9 +6069,11 @@ mod tests {let stream: SendableRecordBatchStream = stream.into();let output = collect(stream).await?;- // Same flush cadence as the column-backed test: one flush per input- // batch, each sliced into batches of 2 and 1 rows.- assert_eq!(output.len(), 2 * num_input_batches);+ // One flush per input batch. The memory limit only fits the table, so+ // holding the materialized flush while slicing it does not fit and each+ // flush is emitted whole instead of in `batch_size` slices.+ assert_eq!(output.len(), num_input_batches);+ assert!(output.iter().all(|batch| batch.num_rows() == 3));assert_snapshot!(batches_to_string(&output), @r"+---+---+-------------+| a | n | SUM(b)[sum] |diff --git a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rsindex d5057d53cf..54479865f1 100644--- a/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs+++ b/datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs@@ -17,22 +17,20 @@//! Partial-reduce hash aggregation stream implementation.-use std::ops::ControlFlow;use std::sync::Arc;-use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::{DataFusionError, Result};-use datafusion_execution::TaskContext;+use datafusion_common::{DataFusionError, Result, internal_datafusion_err};use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_execution::{TaskContext, TryEmitter, async_try_stream};use futures::stream::{Stream, StreamExt};use super::AggregateExec;use super::aggregate_hash_table::{AggregateHashTable, PartialReduceMarker};use crate::metrics::{BaselineMetrics, Count, MetricBuilder, RecordOutput, SpillMetrics};-use crate::stream::EmptyRecordBatchStream;-use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};+use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};+use crate::{InputOrderMode, SendableRecordBatchStream};/// Hash aggregation can combine multiple partial stages before final/// evaluation. This stream implements the partial-reduce stage.@@ -91,88 +89,17 @@ pub(crate) struct PartialReduceHashAggregateStream {/// Number of times accumulated states were emitted due to memory pressure.early_emit_count: Count,- /// Tracks the high-level stream lifecycle. The hash table owns the lower-level- /// state for emitting output batches.- state: Option<PartialReduceHashAggregateState>,-}--/// States for partial-reduce hash aggregation processing.-// The typestate pattern mirrors the final stream and keeps the input/output-// semantics explicit for this mode.-enum PartialReduceHashAggregateState {- ReadingInput {- hash_table: AggregateHashTable<PartialReduceMarker>,- },- /// A fully materialized partial-state batch being emitted incrementally- /// because the table ran out of memory while reading input.- EmittingOnMemoryPressure {- hash_table: AggregateHashTable<PartialReduceMarker>,- // After each incremental emitting step, `remaining_groups` is updated- // with batch slicing.- remaining_groups: RecordBatch,- },- ProducingOutput {- hash_table: AggregateHashTable<PartialReduceMarker>,- },- Done,- /// Sentinel state to use when returning error from any other states, because:- /// - It explicitly releases state-owned resources immediately- /// - More defensive against accidentally resuming execution after error- Error,+ /// The hash table owns the lower-level state for emitting output batches.+ ///+ /// This will be None after the stream is created+ hash_table: Option<AggregateHashTable<PartialReduceMarker>>,}-type PartialReduceHashAggregatePoll = Poll<Option<Result<RecordBatch>>>;-type PartialReduceHashAggregateStateTransition = ControlFlow<- (- PartialReduceHashAggregatePoll,- PartialReduceHashAggregateState,- ),- PartialReduceHashAggregateState,->;--impl PartialReduceHashAggregateState {- fn hash_table(&self) -> &AggregateHashTable<PartialReduceMarker> {- match self {- Self::ReadingInput { hash_table }- | Self::EmittingOnMemoryPressure { hash_table, .. }- | Self::ProducingOutput { hash_table } => hash_table,- Self::Done | Self::Error => {- unreachable!("Done and Error states do not hold a hash table")- }- }- }-- fn hash_table_mut(&mut self) -> &mut AggregateHashTable<PartialReduceMarker> {- match self {- Self::ReadingInput { hash_table }- | Self::EmittingOnMemoryPressure { hash_table, .. }- | Self::ProducingOutput { hash_table } => hash_table,- Self::Done | Self::Error => {- unreachable!("Done and Error states do not hold a hash table")- }- }- }-- fn into_hash_table(self) -> AggregateHashTable<PartialReduceMarker> {- match self {- Self::ReadingInput { hash_table }- | Self::EmittingOnMemoryPressure { hash_table, .. }- | Self::ProducingOutput { hash_table } => hash_table,- Self::Done | Self::Error => {- unreachable!("Done and Error states do not hold a hash table")- }- }- }-- fn into_producing_output(self) -> Self {- Self::ProducingOutput {- hash_table: self.into_hash_table(),- }- }-- fn into_done(self) -> Self {- Self::Done- }+#[derive(PartialEq)]+enum HandleInputResult {+ ProcessNext,+ #[expect(clippy::upper_case_acronyms)]+ OOM,}impl PartialReduceHashAggregateStream {@@ -219,7 +146,7 @@ impl PartialReduceHashAggregateStream {baseline_metrics,reservation,early_emit_count,- state: Some(PartialReduceHashAggregateState::ReadingInput { hash_table }),+ hash_table: Some(hash_table),})}@@ -228,327 +155,505 @@ impl PartialReduceHashAggregateStream {self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));}- fn break_with_err(- error: DataFusionError,- ) -> PartialReduceHashAggregateStateTransition {- ControlFlow::Break((- Poll::Ready(Some(Err(error))),- PartialReduceHashAggregateState::Error,- ))+ pub(crate) fn into_stream(self) -> SendableRecordBatchStream {+ let schema = Arc::clone(&self.schema);++ Box::pin(RecordBatchStreamAdapter::new(schema, self.create_stream()))}- /// Handle ReadingInput state - aggregate partial state batches into the hash table.- ///- /// See comments at `poll_next()` for details.- ///- /// Returns the next operator state with control flow decision.- fn handle_reading_input(- &mut self,- cx: &mut Context<'_>,- mut original_state: PartialReduceHashAggregateState,- ) -> PartialReduceHashAggregateStateTransition {- debug_assert!(matches!(- &original_state,- PartialReduceHashAggregateState::ReadingInput { .. }- ));- debug_assert!(original_state.hash_table().is_building());-- match self.input.poll_next_unpin(cx) {- Poll::Pending => ControlFlow::Break((Poll::Pending, original_state)),- // Get a new input batch, aggregate it in the hash table- Poll::Ready(Some(Ok(batch))) => {- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- let timer = elapsed_compute.timer();- let result = original_state.hash_table_mut().aggregate_batch(&batch);- timer.done();+ /// Entry point for the partial reduce hash aggregate.+ fn create_stream(mut self) -> impl Stream<Item = Result<RecordBatch>> {+ async_try_stream(|mut emitter| async move {+ let mut hash_table: AggregateHashTable<PartialReduceMarker> =+ self.hash_table.take().expect("must have hash table");- if let Err(e) = result {- return Self::break_with_err(e);- }+ debug_assert!(hash_table.is_building());+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- // Update the memory reservation. If OOM, do early emit.- self.resize_or_emit_early(original_state)- }- Poll::Ready(Some(Err(e))) => Self::break_with_err(e),- // Input ends, move to output state- Poll::Ready(None) => {- self.close_input();- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ while let Some(batch) = self.input.next().await.transpose()? {let timer = elapsed_compute.timer();- let result = original_state.hash_table_mut().start_output();- timer.done();- match result {- Ok(()) => {- ControlFlow::Continue(original_state.into_producing_output())+ let state = self.handle_input_batch(&batch, &mut hash_table)?;+ // Avoid holding on the batch+ drop(batch);++ match state {+ HandleInputResult::ProcessNext => {}+ HandleInputResult::OOM => {+ let materialized_group_states = hash_table.take_state_batch()?.ok_or_else(|| {+ internal_datafusion_err!(+ "Partial reduce hash aggregate ran out of memory with no aggregated groups"+ )+ })?;++ self.early_emit_count.add(1);+ timer.done();+ self.emit_on_memory_pressure(+ materialized_group_states,+ &mut emitter,+ hash_table.memory_size(),+ )+ .await?;}- Err(e) => Self::break_with_err(e),}}++ let timer = elapsed_compute.timer();++ self.close_input();+ hash_table.start_output()?;++ timer.done();++ self.produce_output(hash_table, emitter).await?;++ Ok(())+ })+ }++ /// Aggregate partial state batch into the hash table+ fn handle_input_batch(+ &mut self,+ batch: &RecordBatch,+ hash_table: &mut AggregateHashTable<PartialReduceMarker>,+ ) -> Result<HandleInputResult> {+ debug_assert!(hash_table.is_building());+ hash_table.aggregate_batch(batch)?;++ let resize_result = self.reservation.try_resize(hash_table.memory_size());+ match resize_result {+ Ok(()) => Ok(HandleInputResult::ProcessNext),+ Err(DataFusionError::ResourcesExhausted(_)) => Ok(HandleInputResult::OOM),+ Err(e) => Err(e),}}- /// Update the memory reservation. If the reservation succeeds, continue reading- /// input. If OOM, clear the aggregated states in the hash table, and early emit- /// them immediately.- ///- /// Returns the next state; the caller finishes the intended task based on it.- ///- /// The reservation is left at its pre-emission size while the states are being- /// emitted, because the cleared states are still held in memory as- /// `remaining_groups`. The reservation will be reset after exiting the- /// `EmittingOnMemoryPressure` state.+ /// emit a materialized partial-state on memory pressure+ /// batch in `batch_size`(from configuration) slices////// # Implementation Note/// All accumulated states are materialized at once, and then sliced into- /// `batch_size` output batches. Emit them incrementally after blocked state- /// management is ready.+ /// `batch_size` output batches (in case we have enough memory to hold on them while slicing).+ /// Emit them incrementally after blocked state management is ready.////// Issue: <https://github.com/apache/datafusion/issues/7065>- fn resize_or_emit_early(+ async fn emit_on_memory_pressure(&mut self,- mut original_state: PartialReduceHashAggregateState,- ) -> PartialReduceHashAggregateStateTransition {- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- let _timer = elapsed_compute.timer(); // Stop on drop- let resize_result = self+ remaining_groups: RecordBatch,+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,+ hash_table_mem_size: usize,+ ) -> Result<()> {+ let remaining_groups_memory = remaining_groups.get_array_memory_size();++ // Emitting clears the aggregate table and releases its+ // accumulated memory. Update the reservation accordingly.+ // We account here for the remaining groups memory to see if we can return batch size states+ // if there is not enough memory, fallback to emit large batch+ match self.reservation- .try_resize(original_state.hash_table().memory_size());-- let oom = match resize_result {- Ok(()) => return ControlFlow::Continue(original_state),- Err(e @ DataFusionError::ResourcesExhausted(_)) => e,- Err(e) => return Self::break_with_err(e),- };-- let state_batch_result = original_state.hash_table_mut().take_state_batch();-- match state_batch_result {- Ok(Some(remaining_groups)) => {- self.early_emit_count.add(1);- ControlFlow::Continue(- PartialReduceHashAggregateState::EmittingOnMemoryPressure {- hash_table: original_state.into_hash_table(),- remaining_groups,- },- )+ .try_resize(hash_table_mem_size + remaining_groups_memory)+ {+ Ok(_) => {+ // Continue with slicing}- // No accumulated group to emit, so early emission cannot release any- // memory: report the original error.- Ok(None) => Self::break_with_err(oom),- Err(e) => Self::break_with_err(e),+ Err(DataFusionError::ResourcesExhausted(_)) => {+ // Fail to reserve memory for the hash table + state batch while slicing so emit a huge batch++ // Try resize without holding the state batch, if it fails there is nothing we can do+ self.reservation.try_resize(hash_table_mem_size)?;++ emitter+ .emit(remaining_groups.record_output(&self.baseline_metrics))+ .await;++ return Ok(());+ }+ Err(e) => return Err(e),+ }++ let mut index = 0;++ while index + self.batch_size < remaining_groups.num_rows() {+ // More batch to output+ let output = remaining_groups.slice(index, self.batch_size);+ index += self.batch_size;++ emitter+ .emit(output.record_output(&self.baseline_metrics))+ .await;}++ let last_batch =+ remaining_groups.slice(index, remaining_groups.num_rows() - index);++ debug_assert!(last_batch.num_rows() > 0);+ debug_assert!(last_batch.num_rows() <= self.batch_size);++ // We are no longer holding on the batch while slicing, so release the memory.+ // The memory will now equal to the hash table size.+ // Drop our reference first so the generator does not keep the+ // allocation alive (unaccounted) until it is polled again+ drop(remaining_groups);+ self.reservation.try_shrink(remaining_groups_memory)?;++ emitter+ .emit(last_batch.record_output(&self.baseline_metrics))+ .await;++ Ok(())}- /// Handle EmittingOnMemoryPressure state - emit a materialized partial-state- /// batch in `batch_size`(from configuration) slices. After all slices are- /// emitted, update the memory reservation and resume reading input.- ///- /// See comments at `poll_next()` for details.- ///- /// Returns the next operator state with control flow decision.- fn handle_emitting_on_memory_pressure(+ /// Emit merged partial aggregate state batches.+ async fn produce_output(&mut self,- original_state: PartialReduceHashAggregateState,- ) -> PartialReduceHashAggregateStateTransition {- let PartialReduceHashAggregateState::EmittingOnMemoryPressure {- hash_table,- remaining_groups: batch,- } = original_state- else {- unreachable!("expected the EmittingOnMemoryPressure state")- };-- let (output_batch, next_state) = if batch.num_rows() <= self.batch_size {- // Go back to `ReadingInput`- (- batch,- PartialReduceHashAggregateState::ReadingInput { hash_table },+ mut hash_table: AggregateHashTable<PartialReduceMarker>,+ mut emitter: TryEmitter<RecordBatch, DataFusionError>,+ ) -> Result<()> {+ debug_assert!(!hash_table.is_building());++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();++ let mut timer = elapsed_compute.timer();++ loop {+ let Some(batch) = hash_table.next_output_batch()? else {+ // Only reachable when the table held no groups at all: a+ // non-empty table always reports its last batch together with+ // the `Done` state, which the `try_resize` below already zeroes.+ self.reservation.try_resize(0)?;+ return Ok(());+ };++ debug_assert!(batch.num_rows() > 0);++ // The table hands over its groups as they are materialized and+ // reports a size of 0 once it reaches `Done`, so this releases the+ // reservation before the final batch goes downstream.+ // The output is already materialized, so a failed resize cannot+ // be acted on: keep the reservation as is and finish the output.+ let _ = self.reservation.try_resize(hash_table.memory_size());++ timer.done();+ emitter+ .emit(batch.record_output(&self.baseline_metrics))+ .await;+ timer = elapsed_compute.timer();+ }+ }+}++#[cfg(test)]+mod tests {+ use crate::ExecutionPlan;+ use crate::aggregates::partial_reduce_stream::PartialReduceHashAggregateStream;+ use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};+ use crate::test::exec::BarrierExec;+ use arrow::array::{AsArray, Int32Array, Int64Array, RecordBatch};+ use arrow::datatypes::Int32Type;+ use arrow_schema::{DataType, Field, Schema};+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_execution::{SendableRecordBatchStream, TaskContext};+ use datafusion_functions_aggregate::count::count_udaf;+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;+ use datafusion_physical_expr::expressions::col;+ use futures::StreamExt;+ use std::sync::Arc;+ use std::time::Duration;++ /// Builds a partial reduce hash aggregate stream over a single input batch of+ /// `num_groups` distinct groups, running under `memory_limit` bytes.+ ///+ /// The input does not signal end-of-stream until `wait_finish` is called+ /// on the returned [`BarrierExec`], so any output produced before that can+ /// only come from the memory pressure emission path (normal output waits+ /// for all input).+ fn partial_reduce_stream_under_memory_limit(+ memory_limit: usize,+ batch_size: usize,+ num_groups: usize,+ ) -> datafusion_common::Result<(+ SendableRecordBatchStream,+ Arc<BarrierExec>,+ Arc<datafusion_execution::runtime_env::RuntimeEnv>,+ )> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("group_col", DataType::Int32, false),+ Field::new("value_col_state", DataType::Int64, false),+ ]));++ let group_ids: Vec<i32> = (0..num_groups as i32).collect();+ let values: Vec<i64> = vec![1; num_groups];++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(group_ids)),+ Arc::new(Int64Array::from(values)),+ ],+ )?;+ let input_partitions = vec![vec![batch]];++ let runtime = RuntimeEnvBuilder::default()+ .with_memory_limit(memory_limit, 1.0)+ .build_arc()?;++ let mut task_ctx = TaskContext::default().with_runtime(Arc::clone(&runtime));+ let session_config = task_ctx.session_config().clone().set(+ "datafusion.execution.batch_size",+ &datafusion_common::ScalarValue::UInt64(Some(batch_size as u64)),+ );+ task_ctx = task_ctx.with_session_config(session_config);+ let task_ctx = Arc::new(task_ctx);++ // Create aggregate: COUNT(*) GROUP BY group_col+ let group_expr = vec![(col("group_col", &schema)?, "group_col".to_string())];+ let aggr_expr = vec![Arc::new(+ AggregateExprBuilder::new(+ count_udaf(),+ vec![col("value_col_state", &schema)?],)- } else {- // More batches to output, continue in the current state.- let remaining =- batch.slice(self.batch_size, batch.num_rows() - self.batch_size);- let output = batch.slice(0, self.batch_size);- (- output,- PartialReduceHashAggregateState::EmittingOnMemoryPressure {- hash_table,- remaining_groups: remaining,- },+ .schema(Arc::clone(&schema))+ .alias("count_value")+ .build()?,+ )];++ let input = Arc::new(+ BarrierExec::new(input_partitions, Arc::clone(&schema))+ .without_start_barrier()+ .with_finish_barrier()+ .with_log(false),+ );++ let aggregate_exec = AggregateExec::try_new(+ AggregateMode::PartialReduce,+ PhysicalGroupBy::new_single(group_expr),+ aggr_expr,+ vec![None],+ Arc::clone(&input) as Arc<dyn ExecutionPlan>,+ Arc::clone(&schema),+ )?;++ let stream =+ PartialReduceHashAggregateStream::new(&aggregate_exec, &task_ctx, 0)?+ .into_stream();++ Ok((stream, input, runtime))+ }++ #[tokio::test]+ async fn test_partial_reduce_hash_stream_accounts_held_batch_on_memory_pressure_while_slicing()+ -> datafusion_common::Result<()> {+ // When memory pressure triggers early emission, the materialized state+ // batch is held while it is sliced into `batch_size` outputs. The+ // stream must keep that held batch accounted for in its memory+ // reservation until the last slice is emitted; before the fix the+ // reservation was resized down to just the (emptied) hash table size,+ // leaving the held batch unaccounted.++ let batch_size = 1024;+ // One row per group so the state batch is emitted in 4 slices+ let num_groups = 4 * batch_size;++ // Smaller than the building hash table (so pressure triggers) but large+ // enough to hold the materialized state batch (so slicing can proceed)+ let memory_limit = 100 * 1024;+ let (mut stream, input, runtime) = partial_reduce_stream_under_memory_limit(+ memory_limit,+ batch_size,+ num_groups,+ )?;++ // The first output batch must be a pressure-emitted slice, with the rest+ // of the materialized state batch still held by the stream+ let first = tokio::time::timeout(Duration::from_secs(5), stream.next())+ .await+ .expect(+ "did not get early emit due to OOM, this probably means that the \+ memory limit is too high to trigger the OOM",)- };+ .expect("stream ended early")?;+ assert_eq!(first.num_rows(), batch_size);++ // The emitted slice shares buffers with the held state batch, so its+ // array memory size reflects the full held allocation+ let held_size = first.get_array_memory_size();+ let reserved = runtime.memory_pool.reserved();+ assert!(+ reserved >= held_size,+ "memory pool has {reserved} bytes reserved but the stream is \+ holding a materialized state batch of {held_size} bytes"+ );++ let second = stream.next().await.expect("stream ended early")?;+ assert_eq!(second.num_rows(), batch_size);++ // Make sure the state batch is really being sliced (and not emitted whole by the fallback path):+ // the second output must share the same underlying buffer as the first+ //+ // If you changed the code and this fail because+ // - you now deep copy `batch_size` from the full state batch, please update this assertion to something else+ // - you only take batch size from the hash table, you can remove the test+ assert_eq!(+ first+ .column(0)+ .as_primitive::<Int32Type>()+ .values()+ .inner()+ .data_ptr(),+ second+ .column(0)+ .as_primitive::<Int32Type>()+ .values()+ .inner()+ .data_ptr(),+ "both batches should be slices of the same materialized state batch"+ );++ // Let the input finish and drain the stream: no groups lost+ input.wait_finish().await;+ let mut total_rows = first.num_rows() + second.num_rows();+ while let Some(batch) = stream.next().await {+ total_rows += batch?.num_rows();+ }+ assert_eq!(total_rows, num_groups);- debug_assert!(output_batch.num_rows() > 0);- ControlFlow::Break((- Poll::Ready(Some(Ok(output_batch.record_output(&self.baseline_metrics)))),- next_state,- ))+ Ok(())}- /// Handle ProducingOutput state - emit merged partial aggregate state batches.- ///- /// See comments at `poll_next()` for details.- ///- /// Returns the next operator state with control flow decision.- fn handle_producing_output(- &mut self,- mut original_state: PartialReduceHashAggregateState,- ) -> PartialReduceHashAggregateStateTransition {- debug_assert!(matches!(- &original_state,- PartialReduceHashAggregateState::ProducingOutput { .. }- ));- debug_assert!(!original_state.hash_table().is_building());+ #[tokio::test]+ async fn test_partial_reduce_hash_stream_emits_whole_batch_when_held_batch_does_not_fit()+ -> datafusion_common::Result<()> {+ // When memory pressure triggers early emission but the materialized+ // state batch itself does not fit in the reservation, the stream must+ // not fail with a resources exhausted error. Instead it gives up on+ // slicing and emits the whole state batch at once.++ let batch_size = 1024;+ let num_groups = 4 * batch_size;++ // Smaller than the materialized state batch (4096 rows of Int32 group+ // keys plus Int64 counts is at least 48 KiB), so the reservation for+ // hash table held batch fails. The emptied hash table itself is tiny+ // and still fits.+ let memory_limit = 32 * 1024;+ let (mut stream, input, runtime) = partial_reduce_stream_under_memory_limit(+ memory_limit,+ batch_size,+ num_groups,+ )?;- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- let timer = elapsed_compute.timer();- let result = original_state.hash_table_mut().next_output_batch();- timer.done();-- match result {- Ok(Some(batch)) => {- // The output is already materialized, so a failed resize cannot- // be acted on: keep the reservation as is and finish the output.- let _ = self- .reservation- .try_resize(original_state.hash_table().memory_size());- debug_assert!(batch.num_rows() > 0);- let next_state = if original_state.hash_table().is_done() {- original_state.into_done()- } else {- original_state- };-- ControlFlow::Break((- Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),- next_state,- ))- }- Ok(None) => {- let _ = self.reservation.try_resize(0);- ControlFlow::Continue(original_state.into_done())- }- Err(e) => Self::break_with_err(e),+ let first = tokio::time::timeout(Duration::from_secs(5), stream.next())+ .await+ .expect(+ "did not get early emit due to OOM, this probably means that the \+ memory limit is too high to trigger the OOM",+ )+ .expect("stream ended early")?;++ // The whole state batch is emitted at once instead of `batch_size` slices+ assert_eq!(first.num_rows(), num_groups);+ assert!(+ first.get_array_memory_size() > memory_limit,+ "test setup is wrong: the state batch fits within the memory limit, \+ so the slicing path would have been taken"+ );++ // Unlike the slicing path, the stream does not hold on to the emitted+ // batch, so it must not be accounted for in the reservation. Only the+ // (emptied) hash table remains reserved+ let emitted_size = first.get_array_memory_size();+ let reserved = runtime.memory_pool.reserved();+ assert!(+ reserved < emitted_size,+ "memory pool has {reserved} bytes reserved but the stream no longer \+ holds the emitted state batch of {emitted_size} bytes"+ );++ input.wait_finish().await;+ let mut total_rows = first.num_rows();+ while let Some(batch) = stream.next().await {+ total_rows += batch?.num_rows();}+ assert_eq!(total_rows, num_groups);++ Ok(())}-}-impl Stream for PartialReduceHashAggregateStream {- type Item = Result<RecordBatch>;+ #[tokio::test]+ async fn test_partial_reduce_hash_stream_releases_held_batch_after_last_slice()+ -> datafusion_common::Result<()> {+ // While the pressure-emitted state batch is sliced, the stream holds+ // the remaining groups and keeps them reserved. Once the last slice is+ // handed out nothing is held anymore, so the reservation must drop+ // back to just the (emptied) hash table before the input is resumed.++ let batch_size = 1024;+ let num_slices = 4;+ let num_groups = num_slices * batch_size;++ let memory_limit = 100 * 1024;+ let (mut stream, input, runtime) = partial_reduce_stream_under_memory_limit(+ memory_limit,+ batch_size,+ num_groups,+ )?;- /// Entry point for the partial-reduce hash aggregate state machine.- ///- /// See comments in [`PartialReduceHashAggregateStream`] for high-level ideas.- ///- /// State transition graph:- ///- /// ```text- /// (start)- /// -> ReadingInput- /// The stream starts by polling partial-state input and merging those- /// states into the partial-reduce hash table.- ///- /// ReadingInput- /// -> ReadingInput- /// Aggregate one partial-state input batch, update the inner aggregate- /// hash table, and continue with the next input batch.- ///- /// -> EmittingOnMemoryPressure- /// The table cannot reserve enough memory. Materialize all accumulated- /// partial states and begin emitting them incrementally.- ///- /// -> ProducingOutput- /// Input was exhausted. Move to the next state to start outputting- /// merged partial aggregate states.- ///- /// EmittingOnMemoryPressure- /// -> EmittingOnMemoryPressure- /// One batch-sized slice was yielded; repeat until all materialized- /// partial states are emitted.- ///- /// -> ReadingInput- /// The materialized states were emitted; continue with the empty table.- ///- /// ProducingOutput- /// -> ProducingOutput- /// One merged partial-state output batch was yielded; repeat to- /// continue producing output incrementally.- ///- /// -> Done- /// All merged partial-state output was emitted.- ///- /// Any active state- /// -> Error- /// An error drops state-owned resources before it is returned.- ///- /// Error- /// -> (end)- ///- /// Done- /// -> (end)- /// ```- fn poll_next(- mut self: std::pin::Pin<&mut Self>,- cx: &mut Context<'_>,- ) -> Poll<Option<Self::Item>> {- loop {- let cur_state = self- .state- .take()- .expect("PartialReduceHashAggregateStream state should not be None");-- let next_state = match cur_state {- state @ PartialReduceHashAggregateState::ReadingInput { .. } => {- self.handle_reading_input(cx, state)- }- state @ PartialReduceHashAggregateState::EmittingOnMemoryPressure {- ..- } => self.handle_emitting_on_memory_pressure(state),- state @ PartialReduceHashAggregateState::ProducingOutput { .. } => {- self.handle_producing_output(state)- }- state @ PartialReduceHashAggregateState::Error => {- self.close_input();- self.reservation.free();- self.state = Some(state);- return Poll::Ready(None);- }- state @ PartialReduceHashAggregateState::Done => {- let _ = self.reservation.try_resize(0);- self.state = Some(state);- return Poll::Ready(None);- }+ // The input has not finished, so all of these are pressure-emitted slices+ let mut held_size = 0;+ for slice_idx in 0..num_slices {+ let slice = if slice_idx == 0 {+ tokio::time::timeout(Duration::from_secs(5), stream.next())+ .await+ .expect(+ "did not get early emit due to OOM, this probably means that the \+ memory limit is too high to trigger the OOM",+ )+ .expect("stream ended early")?+ } else {+ stream.next().await.expect("stream ended early")?};- match next_state {- ControlFlow::Continue(next_state) => {- self.state = Some(next_state);- }- ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {- debug_assert!(matches!(- next_state,- PartialReduceHashAggregateState::Error- ));-- // The handler has already discarded its state-owned resources.- // Release the remaining stream-owned resources before returning.- self.close_input();- self.reservation.free();- self.state = Some(PartialReduceHashAggregateState::Error);- return Poll::Ready(Some(Err(e)));- }- ControlFlow::Break((poll, next_state)) => {- self.state = Some(next_state);- return poll;- }+ assert_eq!(slice.num_rows(), batch_size);++ // Every slice shares buffers with the held state batch, so this is+ // the size of the full held allocation+ held_size = slice.get_array_memory_size();+ let reserved = runtime.memory_pool.reserved();++ if slice_idx + 1 < num_slices {+ assert!(+ reserved >= held_size,+ "after slice {slice_idx} the stream still holds {held_size} \+ bytes but only {reserved} bytes are reserved"+ );+ } else {+ assert!(+ reserved < held_size,+ "after the last slice nothing is held anymore but {reserved} \+ bytes are still reserved (held batch was {held_size} bytes)"+ );++ // The stream must not keep its own reference to the state batch+ // after handing out the last slice, otherwise the allocation stays+ // alive (but unaccounted) until the stream is polled again+ let buffer = slice.column(0).as_primitive::<Int32Type>().values().inner();+ assert_eq!(+ buffer.strong_count(),+ 1,+ "the stream still holds a reference to the state batch after \+ emitting the last slice"+ );}}- }-}+ assert!(held_size > 0);++ input.wait_finish().await;+ let mut total_rows = num_groups;+ while let Some(batch) = stream.next().await {+ total_rows += batch?.num_rows();+ }+ assert_eq!(total_rows, num_groups);-impl RecordBatchStream for PartialReduceHashAggregateStream {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)+ Ok(())}}