Skip to content

Apache DataFusion SQL Query Engine

perf: Improve output materializing speed for `OrderedFinalAggregateStream` (#25639)

Yongting You88306ff12a86
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
index e4b8118f94..f70fc0bf0d 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
@@ -80,13 +80,24 @@ impl OrderedAggregateTable<FinalMarker> {
)
}
- /// See comments in `ordered_partial_stream::next_output_batch`
- pub(in crate::aggregates) fn next_output_batch(
+ /// Materializes final results for all groups proven complete by the input
+ /// ordering, leaving the active ordered-key range in the table.
+ ///
+ /// Returns None if there are no completed groups.
+ pub(in crate::aggregates) fn take_completed_result_batch(
&mut self,
) -> Result<Option<RecordBatch>> {
- self.next_output_batch_inner(
+ if self.is_empty() {
+ return Ok(None);
+ }
+ let Some(emit_to) = self.group_ordering().emit_to() else {
+ return Ok(None);
+ };
+ self.materialize_groups(
+ emit_to,
HashAggregateAccumulator::evaluate_to_columns,
AccumulatorPhase::Evaluate,
)
+ .map(Some)
}
}
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 3f17cacb52..69568bcda3 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -719,7 +719,7 @@ impl From<StreamType> for SendableRecordBatchStream {
StreamType::FinalHash(stream) => stream.into_stream(),
StreamType::SingleHash(stream) => Box::pin(stream),
StreamType::OrderedPartialAggregate(stream) => stream.into_stream(),
- StreamType::OrderedFinalAggregate(stream) => Box::pin(stream),
+ StreamType::OrderedFinalAggregate(stream) => stream.into_stream(),
StreamType::OrderedSingleAggregate(stream) => Box::pin(stream),
StreamType::GroupedHash(stream) => Box::pin(stream),
StreamType::GroupedPriorityQueue(stream) => Box::pin(stream),
diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
index 3a09dbd97d..4a70ef2df6 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
@@ -17,16 +17,14 @@
//! Final aggregate stream for ordered partial-state input.
-use std::ops::ControlFlow;
use std::sync::Arc;
-use std::task::{Context, Poll};
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::{DataFusionError, Result, internal_err};
-use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
-use futures::stream::{Stream, StreamExt};
+use datafusion_execution::{TaskContext, TryEmitter, async_try_stream};
+use futures::stream::StreamExt;
use super::AggregateExec;
use super::aggregate_hash_table::{
@@ -34,9 +32,9 @@ use super::aggregate_hash_table::{
};
use super::spill::AggregateSpill;
use crate::aggregates::AggregateMode;
-use crate::metrics::{BaselineMetrics, RecordOutput, SpillMetrics};
-use crate::stream::EmptyRecordBatchStream;
-use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};
+use crate::metrics::{BaselineMetrics, SpillMetrics};
+use crate::stream::{ObservedStream, RecordBatchStreamAdapter};
+use crate::{InputOrderMode, SendableRecordBatchStream};
/// Final aggregate stream for `InputOrderMode::Sorted` and
/// `InputOrderMode::PartiallySorted`.
@@ -58,44 +56,38 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};
/// - After input ends, merge the sorted runs and replay them through a fully
/// ordered final aggregate stream.
pub(crate) struct OrderedFinalAggregateStream {
- schema: SchemaRef,
- input: SendableRecordBatchStream,
reservation: MemoryReservation,
- baseline_metrics: BaselineMetrics,
- state: Option<OrderedFinalAggregateState>,
+ context: OrderedFinalAggregateContext,
+ stage: ExecutionStage,
+}
+
+/// Execution stages described in [`OrderedFinalAggregateStream::into_stream`].
+enum ExecutionStage {
+ Aggregating(Aggregating),
+ Outputting(Outputting),
+ MergingSpills(SendableRecordBatchStream),
}
-/// See comments at `poll_next()` for details.
-enum OrderedFinalAggregateState {
- ReadingInput {
- table: OrderedAggregateTable<FinalMarker>,
- /// None if either
- /// - Disk Manager doesn't enable temporary file creation
- /// - The group keys are fully ordered, it's expected to use bounded memory
- spill_context: Option<Box<AggregateSpill>>,
- },
- Spilling {
- table: OrderedAggregateTable<FinalMarker>,
- spill_context: Box<AggregateSpill>,
- },
- ProducingOutput {
- table: OrderedAggregateTable<FinalMarker>,
- },
- PreparingMergeInput {
- table: OrderedAggregateTable<FinalMarker>,
- spill_context: Box<AggregateSpill>,
- },
- MergingSpills {
- stream: SendableRecordBatchStream,
- },
- Done,
+struct Aggregating {
+ input: SendableRecordBatchStream,
+ table: OrderedAggregateTable<FinalMarker>,
+ /// None when temporary files are disabled or all group keys are ordered.
+ spill_context: Option<Box<AggregateSpill>>,
}
-type OrderedFinalAggregatePoll = Poll<Option<Result<RecordBatch>>>;
-type OrderedFinalAggregateStateTransition = ControlFlow<
- (OrderedFinalAggregatePoll, OrderedFinalAggregateState),
- OrderedFinalAggregateState,
->;
+struct Outputting {
+ /// Materialized final results, emitted in slices of `batch_size` rows.
+ batch: RecordBatch,
+ /// Aggregation stage to resume after output; `None` after EOF.
+ resume: Option<Aggregating>,
+}
+
+/// Immutable execution context shared by aggregation and output emission.
+struct OrderedFinalAggregateContext {
+ schema: SchemaRef,
+ batch_size: usize,
+ baseline_metrics: BaselineMetrics,
+}
impl OrderedFinalAggregateStream {
pub fn new(
@@ -202,546 +194,256 @@ impl OrderedFinalAggregateStream {
metrics,
)?;
Ok(Self {
- schema,
- input,
reservation,
- baseline_metrics,
- state: Some(OrderedFinalAggregateState::ReadingInput {
+ context: OrderedFinalAggregateContext {
+ schema,
+ batch_size,
+ baseline_metrics,
+ },
+ stage: ExecutionStage::Aggregating(Aggregating {
+ input,
table,
spill_context,
}),
})
}
- fn close_input(&mut self) {
- let input_schema = self.input.schema();
- self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));
- }
-
- fn break_with_internal_err(message: &str) -> OrderedFinalAggregateStateTransition {
- ControlFlow::Break((
- Poll::Ready(Some(internal_err!("{message}"))),
- OrderedFinalAggregateState::Done,
- ))
- }
-
- /// Reserve memory for the current aggregate table.
- fn reservation_size_for_table(
- table: &OrderedAggregateTable<FinalMarker>,
- spill_context: Option<&AggregateSpill>,
- ) -> usize {
- let table_size = table.memory_size();
- if spill_context.is_some() {
- // See `OrderedFinalAggregateStream` comments for how is it estimated
- table_size.saturating_add(table.num_groups().saturating_mul(size_of::<u32>()))
- } else {
- table_size
- }
- }
-
- /// Consumes one ordered partial-state input batch, then immediately emits
- /// finalized groups if the ordering proves any group is ready.
+ /// Entry point for the ordered final aggregate execution stages.
///
- /// See comments at `poll_next()` for details.
+ /// See [`OrderedFinalAggregateStream`] for high-level ideas.
///
- /// Returns the next operator state with control flow decision.
- fn handle_reading_input(
- &mut self,
- cx: &mut Context<'_>,
- original_state: OrderedFinalAggregateState,
- ) -> OrderedFinalAggregateStateTransition {
- let OrderedFinalAggregateState::ReadingInput {
- mut table,
- spill_context,
- } = original_state
- else {
- return Self::break_with_internal_err(
- "Ordered final aggregate stream expected ReadingInput state",
- );
- };
-
- match self.input.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((
- Poll::Pending,
- OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- },
- )),
- Poll::Ready(Some(Ok(batch))) => {
- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
- let timer = elapsed_compute.timer();
- let result = table.aggregate_batch(&batch);
- timer.done();
-
- if let Err(e) = result {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- },
- ));
- }
-
- // Check memory reservation, and potentially spill.
- let timer = elapsed_compute.timer();
- let resize_result =
- self.reservation
- .try_resize(Self::reservation_size_for_table(
- &table,
- spill_context.as_deref(),
- ));
- timer.done();
- match resize_result {
- Ok(()) => {}
- Err(e @ DataFusionError::ResourcesExhausted(_)) => {
- let Some(spill_context) = spill_context else {
- // `None` means spilling is not supported, see comments
- // at `OrderedFinalAggregateState` for details.
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- ));
- };
- if table.is_empty() {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- ));
- }
- return ControlFlow::Continue(
- OrderedFinalAggregateState::Spilling {
- table,
- spill_context,
- },
- );
+ /// # Stage transition graph:
+ ///
+ /// ```text
+ /// +----[2]----+ +----[5]----+
+ /// | | | |
+ /// v | v |
+ /// +-------------------+ +------------------+
+ /// | | | |
+ /// (start)-[1]->| Aggregating |---[3]-->| Outputting |
+ /// | |<--[6]---| |
+ /// +-------------------+ +------------------+
+ /// | | |
+ /// [8] +------[4]-----+ [7]
+ /// | | |
+ /// v v v
+ /// +-------------------+ +------------------+
+ /// | | | |
+ /// +--->| MergingSpills |---[10-->| Done |-[11]->(end)
+ /// | | | | |
+ /// | +-------------------+ +------------------+
+ /// | |
+ /// +-----[9]------+
+ /// ```
+ ///
+ /// ## Stages
+ ///
+ /// - [`Aggregating`]: Aggregate input batches.
+ /// - [`Outputting`]: Handle materialzing all aggregated input.
+ /// - [`ExecutionStage::MergingSpills`]: If OOM and spilled before, use this
+ /// stage to finish execution.
+ ///
+ /// ### Incremental output
+ ///
+ /// See the [ordered partial aggregate notes] for details.
+ ///
+ /// [ordered partial aggregate notes]: super::ordered_partial_stream::OrderedPartialAggregateStream::into_stream
+ ///
+ /// ## Transition Edges
+ ///
+ /// 1. Start.
+ /// 2. Merge one input batch:
+ /// - If memory fits and no groups are complete, continue reading input.
+ /// - If OOM, spill.
+ /// 3. Prepare output:
+ /// - Before any spill, ordering proves a prefix complete: materialize the
+ /// entire prefix once, retaining the input and active groups to resume
+ /// aggregation.
+ /// - At EOF without spills, materialize all remaining results and prepare
+ /// to output.
+ /// 4. Input was exhausted, directly end.
+ /// 5. Incremental output at `batch_size`
+ /// 6. The batch was fully emitted and retained aggregation can resume.
+ /// 7. The output batch was fully emitted.
+ /// 8. Input was exhausted after spilling.
+ /// 9. Incremental output during reading spill and finalizing results.
+ /// 10. The merged spill input was fully aggregated and emitted.
+ /// 11. End.
+ pub(crate) fn into_stream(self) -> SendableRecordBatchStream {
+ let Self {
+ reservation,
+ context,
+ stage,
+ } = self;
+ let schema = Arc::clone(&context.schema);
+ let metrics = context.baseline_metrics.clone();
+ let stream = async_try_stream(|mut emitter| async move {
+ let mut stage = Some(stage);
+ while let Some(current_stage) = stage {
+ stage = match current_stage {
+ ExecutionStage::Aggregating(aggregating) => {
+ aggregating.handle_stage(&context, &reservation).await?
}
- Err(e) => {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- ));
+ ExecutionStage::Outputting(outputting) => {
+ outputting
+ .handle_stage(&context, &reservation, &mut emitter)
+ .await?
}
- }
-
- let result = if spill_context
- .as_ref()
- .is_some_and(|spill_context| spill_context.has_spills())
- {
- // Once one incomplete run is spilled, every remaining state
- // must participate in replay so no group is finalized twice.
- Ok(None)
- } else {
- let timer = elapsed_compute.timer();
- let result = table.next_output_batch();
- timer.done();
- result
- };
-
- match result {
- // Some finalized groups can be emitted. Yield them, then
- // continue aggregating input in the current state.
- Ok(Some(batch)) => {
- if let Err(e) =
- self.reservation
- .try_resize(Self::reservation_size_for_table(
- &table,
- spill_context.as_deref(),
- ))
- {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- ));
+ ExecutionStage::MergingSpills(mut stream) => {
+ while let Some(batch) = stream.next().await.transpose()? {
+ emitter.emit(batch).await;
}
- let next_state = OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- };
-
- ControlFlow::Break((
- Poll::Ready(Some(Ok(
- batch.record_output(&self.baseline_metrics)
- ))),
- next_state,
- ))
+ None
}
- // Can't do early emit, continue aggregating.
- Ok(None) => {
- ControlFlow::Continue(OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- })
- }
- Err(e) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- },
- )),
- }
- }
- Poll::Ready(Some(Err(e))) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context,
- },
- )),
- Poll::Ready(None) => {
- self.close_input();
- match spill_context {
- Some(spill_context) if spill_context.has_spills() => {
- ControlFlow::Continue(
- OrderedFinalAggregateState::PreparingMergeInput {
- table,
- spill_context,
- },
- )
- }
- _ => {
- table.input_done();
- ControlFlow::Continue(
- OrderedFinalAggregateState::ProducingOutput { table },
- )
- }
- }
+ };
}
- }
+ Ok(())
+ });
+ let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+ Box::pin(ObservedStream::new(stream, metrics, None))
}
+}
- /// Sorts and spills one complete in-memory state run, then resumes input.
- ///
- /// See comments at `poll_next()` for details.
- ///
- /// Returns the next operator state with control flow decision.
- fn handle_spilling(
- &mut self,
- original_state: OrderedFinalAggregateState,
- ) -> OrderedFinalAggregateStateTransition {
- let OrderedFinalAggregateState::Spilling {
- mut table,
- mut spill_context,
- } = original_state
- else {
- return Self::break_with_internal_err(
- "Ordered final aggregate stream expected Spilling state",
- );
- };
-
- // Sanity check: it's impossible to OOM when the table is empty
- if table.is_empty() {
- return ControlFlow::Break((
- Poll::Ready(Some(internal_err!(
- "Ordered final aggregation entered Spilling with an empty table"
- ))),
- OrderedFinalAggregateState::Done,
- ));
- }
-
- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
- let timer = elapsed_compute.timer();
- let mut result = table
- .take_state_batch()
- .and_then(|batch| spill_context.sort_and_spill(batch));
-
- // Spilling shrinks the aggregate table and releases its accumulated
- // memory. Update the reservation accordingly.
- if let Err(e) = self.reservation.try_resize(table.memory_size()) {
- result =
- Err(e.context("Decreasing allocation after spilling should succeed"));
- }
-
- timer.done();
-
- match result {
- // Finished spilling the aggregate table, continue aggregating from input
- Ok(()) => ControlFlow::Continue(OrderedFinalAggregateState::ReadingInput {
- table,
- spill_context: Some(spill_context),
- }),
- Err(e) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- )),
+impl Aggregating {
+ /// Reserve the table footprint and, when spillable, one sort index per group.
+ fn reservation_size(&self) -> usize {
+ let table_size = self.table.memory_size();
+ if self.spill_context.is_some() {
+ // See `OrderedFinalAggregateStream` for the spill memory estimate.
+ table_size
+ .saturating_add(self.table.num_groups().saturating_mul(size_of::<u32>()))
+ } else {
+ table_size
}
}
- /// 1. Spills the last in-memory run.
- /// 2. Constructs a globally ordered input stream by applying a sort-preserving
- /// merge to all spills.
- /// 3. Constructs a replay stream: an ordered aggregate stream over the fully
- /// ordered input constructed from the spills.
- ///
- /// See comments at `poll_next()` for details.
- ///
- /// Returns the next operator state with control flow decision.
- fn handle_preparing_merge_input(
- &mut self,
- original_state: OrderedFinalAggregateState,
- ) -> OrderedFinalAggregateStateTransition {
- let OrderedFinalAggregateState::PreparingMergeInput {
- mut table,
- mut spill_context,
- } = original_state
- else {
- return Self::break_with_internal_err(
- "Ordered final aggregate stream expected PreparingMergeInput state",
- );
- };
-
- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
- let timer = elapsed_compute.timer();
- let replay = match table
- .take_state_batch()
- .and_then(|batch| spill_context.sort_and_spill(batch))
- {
- Ok(()) => {
- let metrics = table.metrics();
- drop(table);
- match self.reservation.try_resize(0) {
- Ok(()) => (*spill_context).into_replay_stream(
- &self.baseline_metrics,
- metrics,
- self.reservation.new_empty(),
- ),
- Err(e) => Err(e),
+ /// Merges partial states until final results are ready or spill replay begins.
+ async fn handle_stage(
+ mut self,
+ context: &OrderedFinalAggregateContext,
+ reservation: &MemoryReservation,
+ ) -> Result<Option<ExecutionStage>> {
+ let elapsed_compute = context.baseline_metrics.elapsed_compute();
+
+ while let Some(batch) = self.input.next().await.transpose()? {
+ let timer = elapsed_compute.timer();
+ self.table.aggregate_batch(&batch)?;
+
+ match reservation.try_resize(self.reservation_size()) {
+ Ok(()) => {}
+ Err(oom @ DataFusionError::ResourcesExhausted(_)) => {
+ let Some(spill_context) = self.spill_context.as_mut() else {
+ return Err(oom);
+ };
+ if self.table.is_empty() {
+ return Err(oom);
+ }
+ spill_context.sort_and_spill(self.table.take_state_batch()?)?;
+ reservation
+ .try_resize(self.table.memory_size())
+ .map_err(|e| {
+ e.context(
+ "Decreasing allocation after spilling should succeed",
+ )
+ })?;
+ continue;
}
+ Err(e) => return Err(e),
}
- Err(e) => Err(e),
- };
- timer.done();
- match replay {
- Ok(stream) => {
- ControlFlow::Continue(OrderedFinalAggregateState::MergingSpills {
- stream,
- })
+ if self.spill_context.as_ref().is_some_and(|s| s.has_spills()) {
+ // Spilled groups may recur, so all remaining states must go
+ // through replay before any more final results can be emitted.
+ continue;
}
- Err(e) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- )),
+ let Some(batch) = self.table.take_completed_result_batch()? else {
+ continue;
+ };
+ timer.done();
+ return Ok(Some(ExecutionStage::Outputting(Outputting {
+ batch,
+ resume: Some(self),
+ })));
}
- }
- /// Forwards output from the fully ordered stream that consumes the merged
- /// spill runs.
- ///
- /// See comments at `poll_next()` for details.
- ///
- /// Returns the next operator state with control flow decision.
- fn handle_merging_spills(
- &mut self,
- cx: &mut Context<'_>,
- original_state: OrderedFinalAggregateState,
- ) -> OrderedFinalAggregateStateTransition {
- let OrderedFinalAggregateState::MergingSpills { mut stream } = original_state
- else {
- return Self::break_with_internal_err(
- "Ordered final aggregate stream expected MergingSpills state",
- );
- };
-
- match stream.poll_next_unpin(cx) {
- Poll::Pending => ControlFlow::Break((
- Poll::Pending,
- OrderedFinalAggregateState::MergingSpills { stream },
- )),
- Poll::Ready(Some(Ok(batch))) => ControlFlow::Break((
- Poll::Ready(Some(Ok(batch))),
- OrderedFinalAggregateState::MergingSpills { stream },
- )),
- Poll::Ready(Some(Err(e))) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- )),
- Poll::Ready(None) => ControlFlow::Continue(OrderedFinalAggregateState::Done),
+ // Release upstream resources before materializing output or replaying spills.
+ drop(self.input);
+ let timer = elapsed_compute.timer();
+ if let Some(mut spill_context) = self.spill_context.filter(|s| s.has_spills()) {
+ spill_context.sort_and_spill(self.table.take_state_batch()?)?;
+ let metrics = self.table.metrics();
+ drop(self.table);
+ reservation.try_resize(0)?;
+ // The outer ObservedStream counts output; replay only shares compute time.
+ let stream = (*spill_context).into_replay_stream(
+ &context.baseline_metrics.intermediate(),
+ metrics,
+ reservation.new_empty(),
+ )?;
+ timer.done();
+ return Ok(Some(ExecutionStage::MergingSpills(stream)));
}
- }
-
- /// Emits one batch after input is exhausted.
- ///
- /// `table.input_done()` has already made every remaining group safe to emit,
- /// so this state keeps draining until the table is empty.
- ///
- /// See comments at `poll_next()` for details.
- ///
- /// Returns the next operator state with control flow decision.
- fn handle_producing_output(
- &mut self,
- original_state: OrderedFinalAggregateState,
- ) -> OrderedFinalAggregateStateTransition {
- let OrderedFinalAggregateState::ProducingOutput { table } = original_state else {
- return Self::break_with_internal_err(
- "Ordered final aggregate stream expected ProducingOutput state",
- );
- };
- let mut table = table;
- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
- let timer = elapsed_compute.timer();
- let result = table.next_output_batch();
+ self.table.input_done();
+ let output = self.table.take_completed_result_batch()?;
+ drop(self.table);
timer.done();
-
- match result {
- Ok(Some(batch)) => {
- let next_state = if table.is_empty() {
- drop(table);
- if let Err(e) = self.reservation.try_resize(0) {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::Done,
- ));
- }
- OrderedFinalAggregateState::Done
- } else {
- if let Err(e) = self.reservation.try_resize(table.memory_size()) {
- return ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ProducingOutput { table },
- ));
- }
- OrderedFinalAggregateState::ProducingOutput { table }
- };
-
- ControlFlow::Break((
- Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),
- next_state,
- ))
- }
- Err(e) => ControlFlow::Break((
- Poll::Ready(Some(Err(e))),
- OrderedFinalAggregateState::ProducingOutput { table },
- )),
- Ok(None) => {
- drop(table);
- let next_state = OrderedFinalAggregateState::Done;
- if let Err(e) = self.reservation.try_resize(0) {
- return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state));
- }
- ControlFlow::Continue(next_state)
- }
- }
+ let Some(batch) = output else {
+ reservation.try_resize(0)?;
+ return Ok(None);
+ };
+ Ok(Some(ExecutionStage::Outputting(Outputting {
+ batch,
+ resume: None,
+ })))
}
}
-impl Stream for OrderedFinalAggregateStream {
- type Item = Result<RecordBatch>;
-
- /// Entry point for the ordered final aggregate state machine.
- ///
- /// See comments in [`OrderedFinalAggregateStream`] for high-level ideas.
- ///
- /// State transition graph:
- ///
- /// ```text
- /// (start)
- /// -> ReadingInput
- /// The stream starts by polling ordered partial-state input and merging
- /// those states into the ordered final aggregate table.
- ///
- /// ReadingInput
- /// -> ReadingInput
- /// Merge one input batch. If it fits in memory, optionally yield groups
- /// proven complete by the input ordering, then read the next batch.
- /// -> Spilling
- /// The table cannot reserve enough memory. Move all current states into
- /// one fully group-key-sorted spill run.
- /// -> ProducingOutput
- /// Input was exhausted without spilling. Mark every remaining group as
- /// complete and produce its final result.
- /// -> PreparingMergeInput
- /// Input was exhausted after spilling. Spill the last in-memory run and
- /// construct the ordered input used to merge all spill files.
- ///
- /// Spilling
- /// -> ReadingInput
- /// One sorted run was written; resume reading the original input.
- ///
- /// PreparingMergeInput
- /// Spill the final in-memory run and build the input ordered replay stream.
- /// -> MergingSpills
- /// The final run was spilled and the ordered replay stream was built.
- ///
- /// MergingSpills
- /// Aggregate the merged spill runs and emit final results.
- /// -> MergingSpills
- /// Forward one result batch from the fully ordered replay stream that
- /// consumes the sort-preserving merge.
- /// -> Done
- /// The merged spill input was fully aggregated.
- ///
- /// ProducingOutput
- /// -> ProducingOutput
- /// One remaining final aggregate batch was yielded; repeat to continue
- /// draining the table.
- /// -> Done
- /// All remaining groups were emitted.
- ///
- /// Done
- /// -> (end)
- /// ```
- fn poll_next(
- mut self: std::pin::Pin<&mut Self>,
- cx: &mut Context<'_>,
- ) -> Poll<Option<Self::Item>> {
- loop {
- let cur_state = self
- .state
- .take()
- .expect("OrderedFinalAggregateStream state should not be None");
-
- let next_state = match cur_state {
- state @ OrderedFinalAggregateState::ReadingInput { .. } => {
- self.handle_reading_input(cx, state)
- }
- state @ OrderedFinalAggregateState::Spilling { .. } => {
- self.handle_spilling(state)
- }
- state @ OrderedFinalAggregateState::PreparingMergeInput { .. } => {
- self.handle_preparing_merge_input(state)
- }
- state @ OrderedFinalAggregateState::MergingSpills { .. } => {
- self.handle_merging_spills(cx, state)
- }
- state @ OrderedFinalAggregateState::ProducingOutput { .. } => {
- self.handle_producing_output(state)
- }
- state @ OrderedFinalAggregateState::Done => {
- let _ = self.reservation.try_resize(0);
- self.state = Some(state);
- return Poll::Ready(None);
- }
- };
-
- match next_state {
- ControlFlow::Continue(next_state) => {
- self.state = Some(next_state);
- }
- ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {
- // Errors are terminal: discard all operator state and release
- // its upstream input and memory reservation before returning.
- drop(next_state);
- self.close_input();
- self.reservation.free();
- self.state = Some(OrderedFinalAggregateState::Done);
- return Poll::Ready(Some(Err(e)));
- }
- ControlFlow::Break((poll, next_state)) => {
- self.state = Some(next_state);
- return poll;
- }
+impl Outputting {
+ /// Emits slices of one materialized batch without touching the aggregate table.
+ async fn handle_stage(
+ self,
+ context: &OrderedFinalAggregateContext,
+ reservation: &MemoryReservation,
+ emitter: &mut TryEmitter<RecordBatch, DataFusionError>,
+ ) -> Result<Option<ExecutionStage>> {
+ let Self { mut batch, resume } = self;
+ let elapsed_compute = context.baseline_metrics.elapsed_compute();
+ let mut timer = elapsed_compute.timer();
+ let (table_memory, next_stage) = match resume {
+ Some(aggregating) => (
+ aggregating.reservation_size(),
+ Some(ExecutionStage::Aggregating(aggregating)),
+ ),
+ None => (0, None),
+ };
+ let batch_memory = batch.get_array_memory_size();
+ match reservation.try_resize(table_memory + batch_memory) {
+ Ok(()) => {}
+ Err(DataFusionError::ResourcesExhausted(_)) => {
+ // If we cannot hold the batch while slicing, hand it off whole.
+ reservation.try_resize(table_memory)?;
+ timer.done();
+ emitter.emit(batch).await;
+ return Ok(next_stage);
}
+ Err(e) => return Err(e),
+ }
+
+ while batch.num_rows() > context.batch_size {
+ let output = batch.slice(0, context.batch_size);
+ batch =
+ batch.slice(context.batch_size, batch.num_rows() - context.batch_size);
+ timer.done();
+ emitter.emit(output).await;
+ timer = elapsed_compute.timer();
}
- }
-}
-impl RecordBatchStream for OrderedFinalAggregateStream {
- fn schema(&self) -> SchemaRef {
- Arc::clone(&self.schema)
+ // The final slice transfers ownership of the buffers to the consumer.
+ reservation.try_shrink(batch_memory)?;
+ timer.done();
+ emitter.emit(batch).await;
+ Ok(next_stage)
}
}
@@ -751,7 +453,6 @@ mod tests {
use crate::ExecutionPlan;
use crate::aggregates::PhysicalGroupBy;
use crate::common::collect;
- use crate::stream::RecordBatchStreamAdapter;
use crate::test::TestMemoryExec;
use arrow::array::{Int64Array, StringViewArray};
use arrow::datatypes::{DataType, Field, Schema};
@@ -888,7 +589,7 @@ mod tests {
aggregate.input_order_mode(),
)?;
senders.push(sender);
- streams.push(stream);
+ streams.push(stream.into_stream());
}
let mut expected = BTreeMap::new();
let mut make_batch = |partition: i64, start: i64| {
@@ -937,7 +638,7 @@ mod tests {
.unwrap();
assert!(streams[1].next().now_or_never().is_none());
}
- let held = streams[1].reservation.size();
+ let held = pool.reserved();
assert!(held > 500 * 1024);
for batch in 0..input_batches {
// Repeated keys cross spill runs, so replay must merge their sums.
@@ -953,10 +654,10 @@ mod tests {
match finish {
Finish::Collect => {
senders[0].close_channel();
- let mut output = collect(Box::pin(first)).await?;
+ let mut output = collect(first).await?;
assert_eq!(pool.reserved(), held);
senders[1].close_channel();
- output.extend(collect(Box::pin(streams.remove(0))).await?);
+ output.extend(collect(streams.remove(0)).await?);
let mut actual = BTreeMap::new();
for batch in output {
let a = batch
@@ -998,11 +699,7 @@ mod tests {
}
Finish::DropDuringMerge => {
senders[0].close_channel();
- let _ = first.next().now_or_never();
- assert!(matches!(
- first.state.as_ref(),
- Some(OrderedFinalAggregateState::MergingSpills { .. })
- ));
+ assert!(first.next().now_or_never().is_none());
drop(first);
}
Finish::DropDuringReplay => {
diff --git a/datafusion/physical-plan/src/aggregates/spill.rs b/datafusion/physical-plan/src/aggregates/spill.rs
index dea4fe1efd..a1d52382de 100644
--- a/datafusion/physical-plan/src/aggregates/spill.rs
+++ b/datafusion/physical-plan/src/aggregates/spill.rs
@@ -288,6 +288,6 @@ impl AggregateSpill {
None,
reservation,
)?;
- Ok(Box::pin(replay))
+ Ok(replay.into_stream())
}
}