Apache DataFusion SQL Query Engine
refactor(hash-aggr): Support spilling for ordered aggregation (#23657)
Yongting You461dc6d67793
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rsindex c83303c51d..2293e7b1b8 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs@@ -81,6 +81,10 @@ pub(in crate::aggregates) struct OrderedAggregateTable<OrderedAggrMode> {/// Output schema: group columns followed by aggregate state or final values.pub(super) output_schema: SchemaRef,+ /// Intermediate-state schema used when memory pressure requires the table+ /// to pass through or spill its current state.+ pub(super) state_schema: SchemaRef,+/// Maximum rows per emitted output batch, from config `batch_size`.pub(super) batch_size: usize,@@ -129,13 +133,14 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {)]pub(super) fn new_for_mode(agg: &AggregateExec,- partition: usize,input_schema: &SchemaRef,output_schema: SchemaRef,+ state_schema: SchemaRef,batch_size: usize,input_order_mode: &InputOrderMode,aggregate_mode: &AggregateMode,filters: Vec<Option<Arc<dyn PhysicalExpr>>>,+ group_by_metrics: GroupByMetrics,) -> Result<Self> {assert_or_internal_err!(batch_size > 0,@@ -168,8 +173,9 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {Ok(Self {output_schema,+ state_schema,batch_size,- group_by_metrics: GroupByMetrics::new(&agg.metrics, partition),+ group_by_metrics,buffer: OrderedAggregateTableBuffer {group_by: Arc::clone(&agg.group_by),group_ordering,@@ -217,9 +223,19 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {self.buffer.group_ordering.input_done();}+ /// Returns the ordering state used to decide how memory pressure is handled.+ pub(in crate::aggregates) fn group_ordering(&self) -> &GroupOrdering {+ &self.buffer.group_ordering+ }++ /// Number of groups currently buffered.+ pub(in crate::aggregates) fn num_groups(&self) -> usize {+ self.buffer.group_values.len()+ }+/// Check if there is zero groups accumulated so far.pub(in crate::aggregates) fn is_empty(&self) -> bool {- self.buffer.group_values.is_empty()+ self.num_groups() == 0}/// All internal buffer's memory size.@@ -234,6 +250,43 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {+ self.buffer.group_indices.allocated_size()}+ pub(in crate::aggregates) fn group_by_metrics(&self) -> GroupByMetrics {+ self.group_by_metrics.clone()+ }++ /// Takes every intermediate aggregate state and resets the table so it can+ /// continue with a new ordered input segment.+ ///+ /// Unlike normal ordered emission, this operation is allowed to take the+ /// active (incomplete) groups. Partial aggregation can pass those states to+ /// its final stage, while final aggregation sorts and spills them before+ /// replay.+ pub(in crate::aggregates) fn take_state_batch(+ &mut self,+ ) -> Result<Option<RecordBatch>> {+ if self.buffer.group_values.is_empty() {+ return Ok(None);+ }++ let mut output = self.buffer.group_values.emit(EmitTo::All)?;+ for acc in &mut self.buffer.accumulators {+ output.extend(acc.state(EmitTo::All)?);+ }++ let batch = RecordBatch::try_new(Arc::clone(&self.state_schema), output)?;+ debug_assert!(batch.num_rows() > 0);++ // `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the+ // key/index buffers too so the memory reservation can be released+ // before the batch is passed downstream or sorted for spilling.+ self.buffer.group_values.clear_shrink(0);+ self.buffer.group_indices.clear();+ self.buffer.group_indices.shrink_to_fit();+ self.buffer.group_ordering.reset();++ Ok(Some(batch))+ }+/// Returns the [`EmitTo`], clamped to the specified batch size////// Returns `(emit_to, should_remove_groups)`, where `emit_to` is the numberdiff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rsindex b7e3fd38ed..fd064ebffe 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs@@ -19,12 +19,15 @@//!//! See comments in [`super::ordered_partial_table`] for details.+use std::sync::Arc;+use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;use crate::InputOrderMode;use crate::aggregates::aggregate_hash_table::FinalMarker;+use crate::aggregates::group_values::GroupByMetrics;use crate::aggregates::{AggregateExec, AggregateMode};use super::common_ordered::OrderedAggregateTable;@@ -41,21 +44,22 @@ use super::common_ordered::OrderedAggregateTable;impl OrderedAggregateTable<FinalMarker> {pub(in crate::aggregates) fn new_with_input_order(agg: &AggregateExec,- partition: usize,input_schema: &SchemaRef,output_schema: SchemaRef,batch_size: usize,input_order_mode: &InputOrderMode,+ group_by_metrics: GroupByMetrics,) -> Result<Self> {Self::new_for_mode(agg,- partition,input_schema,output_schema,+ Arc::clone(input_schema),batch_size,input_order_mode,&AggregateMode::Final,vec![None; agg.aggr_expr.len()],+ group_by_metrics,)}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rsindex 033c14056a..a04e4dda8f 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs@@ -29,12 +29,15 @@//! The implementation is separated from other aggregate tables because this//! execution path is likely to be optimized further in the future.+use std::sync::Arc;+use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;use crate::aggregates::{AggregateExec, AggregateMode, aggregate_hash_table::PartialMarker,+ group_values::GroupByMetrics,};use super::common_ordered::OrderedAggregateTable;@@ -56,15 +59,18 @@ impl OrderedAggregateTable<PartialMarker> {batch_size: usize,) -> Result<Self> {let input_schema = agg.input().schema();+ let state_schema = Arc::clone(&output_schema);+ let group_by_metrics = GroupByMetrics::new(&agg.metrics, partition);Self::new_for_mode(agg,- partition,&input_schema,output_schema,+ state_schema,batch_size,&agg.input_order_mode,&AggregateMode::Partial,agg.filter_expr.iter().cloned().collect(),+ group_by_metrics,)}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 30e0ad2469..7d93cd7398 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1235,12 +1235,10 @@ impl AggregateExec {&& self.limit_options_supported_by_hash_stream()}- fn should_use_ordered_partial_aggregate_stream(&self, context: &TaskContext) -> bool {- // TODO: implement memory-limited path and remove this limitation- if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {- return false;- }-+ fn should_use_ordered_partial_aggregate_stream(+ &self,+ _context: &TaskContext,+ ) -> bool {self.mode == AggregateMode::Partial&& self.input_order_mode != InputOrderMode::Linear&& !self.group_by.is_true_no_grouping()@@ -1291,12 +1289,7 @@ impl AggregateExec {&& self.group_by.is_single()}- fn should_use_ordered_final_aggregate_stream(&self, context: &TaskContext) -> bool {- // TODO: implement memory-limited path and remove this limitation- if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {- return false;- }-+ fn should_use_ordered_final_aggregate_stream(&self, _context: &TaskContext) -> bool {matches!(self.mode,AggregateMode::Final | AggregateMode::FinalPartitioned@@ -3922,10 +3915,10 @@ mod tests {+----------+-----------+-------------------------+");- // Ordered streams don't implement memory limits yet.+ // Ordered partial aggregation supports finite memory.let finite_memory_task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;let stream = aggregate.execute_typed(0, &finite_memory_task_ctx)?;- assert!(matches!(stream, StreamType::GroupedHash(_)));+ assert!(matches!(stream, StreamType::OrderedPartialAggregate(_)));Ok(())}@@ -3999,10 +3992,10 @@ mod tests {+-----+--------------+");- // Ordered streams don't implement memory limits yet.+ // Ordered final aggregation supports finite memory.let finite_memory_task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;let stream = final_aggregate.execute_typed(0, &finite_memory_task_ctx)?;- assert!(matches!(stream, StreamType::GroupedHash(_)));+ assert!(matches!(stream, StreamType::OrderedFinalAggregate(_)));Ok(())}diff --git a/datafusion/physical-plan/src/aggregates/order/full.rs b/datafusion/physical-plan/src/aggregates/order/full.rsindex eb98611f79..ca818d6a2d 100644--- a/datafusion/physical-plan/src/aggregates/order/full.rs+++ b/datafusion/physical-plan/src/aggregates/order/full.rs@@ -115,6 +115,11 @@ impl GroupOrderingFull {self.state = State::Complete;}+ /// Starts tracking a new fully ordered input segment.+ pub fn reset(&mut self) {+ self.state = State::Start;+ }+/// Called when new groups are added in a batch. See documentation/// on [`super::GroupOrdering::new_groups`]pub fn new_groups(&mut self, total_num_groups: usize) {diff --git a/datafusion/physical-plan/src/aggregates/order/mod.rs b/datafusion/physical-plan/src/aggregates/order/mod.rsindex 97fbd519c8..259411b00b 100644--- a/datafusion/physical-plan/src/aggregates/order/mod.rs+++ b/datafusion/physical-plan/src/aggregates/order/mod.rs@@ -93,6 +93,20 @@ impl GroupOrdering {}}+ /// Resets the ordering state while preserving the configured ordering mode.+ ///+ /// Ordered partial aggregation uses this after passing intermediate states+ /// downstream, and ordered final aggregation uses it after spilling a run.+ /// In both cases the hash table is empty and can start tracking the next+ /// input batch from a fresh ordering state.+ pub fn reset(&mut self) {+ match self {+ GroupOrdering::None => {}+ GroupOrdering::Partial(partial) => partial.reset(),+ GroupOrdering::Full(full) => full.reset(),+ }+ }+/// Removes the first `n` groups from the internal state, shifting all/// existing indexes down by `n`.pub fn remove_groups(&mut self, n: usize) {diff --git a/datafusion/physical-plan/src/aggregates/order/partial.rs b/datafusion/physical-plan/src/aggregates/order/partial.rsindex 476551a7ca..1603bb6d07 100644--- a/datafusion/physical-plan/src/aggregates/order/partial.rs+++ b/datafusion/physical-plan/src/aggregates/order/partial.rs@@ -186,6 +186,12 @@ impl GroupOrderingPartial {};}+ /// Starts tracking a new ordered input segment with the same sort-key+ /// columns.+ pub fn reset(&mut self) {+ self.state = State::Start;+ }+fn updated_sort_key(current_sort: usize,sort_key: Option<Vec<ScalarValue>>,diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rsindex 89653e05ab..071c1e9011 100644--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs@@ -23,22 +23,44 @@ use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::Result;+use datafusion_common::{DataFusionError, Result, internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr_common::sort_expr::LexOrdering;use futures::stream::{Stream, StreamExt};use super::AggregateExec;use super::aggregate_hash_table::{FinalMarker, OrderedAggregateTable};+use super::group_values::GroupByMetrics;use crate::aggregates::AggregateMode;use crate::metrics::{BaselineMetrics, RecordOutput, SpillMetrics};+use crate::sorts::IncrementalSortIterator;+use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};+use crate::spill::spill_manager::SpillManager;use crate::stream::EmptyRecordBatchStream;use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};/// Final aggregate stream for `InputOrderMode::Sorted` and/// `InputOrderMode::PartiallySorted`.///-/// See comments at [`super::ordered_partial_stream`] for details.+/// See comments at [`super::ordered_partial_stream::OrderedPartialAggregateStream`] for details.+///+/// # Spilling+///+/// This section is only for implementation notes, for background, see [`super::ordered_partial_stream::OrderedPartialAggregateStream`]+///+/// For partially sorted input, spilling works as follows:+///+/// - Reserve the table footprint plus one `u32` sort index per buffered group. The+/// extra index array is used in later sorting before spilling.+/// - On memory pressure, materialize all group states into one batch.+/// - Use [`IncrementalSortIterator`] to compute the full-batch index, then+/// materialize and write one sorted `batch_size` slice at a time. The original+/// batch and full index remain live until the run is written.+/// - After input ends, merge the sorted runs and replay them through a fully+/// ordered final aggregate stream.pub(crate) struct OrderedFinalAggregateStream {schema: SchemaRef,input: SendableRecordBatchStream,@@ -47,14 +69,50 @@ pub(crate) struct OrderedFinalAggregateStream {state: Option<OrderedFinalAggregateState>,}+/// Spill configuration and accumulated runs for partially ordered final+/// aggregation.+///+/// Each spill event drains all currently buffered groups, sorts their intermediate+/// states by the full group key, and writes them to one spill file. All files are+/// merged and replayed after the original input ends.+struct OrderedFinalSpillContext {+ /// Aggregate configuration+ agg: AggregateExec,+ /// Task context+ context: Arc<TaskContext>,+ /// Original partition index+ partition: usize,+ /// Target batch size from configuration+ batch_size: usize,+ /// Full group-key ordering, such ordering with be kept in: a) individual spill+ /// files, b) order after final merging and streaming aggregate+ spill_expr: LexOrdering,+ /// Spill I/O and metrics manager.+ spill_manager: SpillManager,+ /// Fully sorted spill runs waiting to be merged.+ spills: Vec<SortedSpillFile>,+}+/// See comments at `poll_next()` for details.enum OrderedFinalAggregateState {ReadingInput {table: OrderedAggregateTable<FinalMarker>,+ spill_context: Option<Box<OrderedFinalSpillContext>>,+ },+ Spilling {+ table: OrderedAggregateTable<FinalMarker>,+ spill_context: Box<OrderedFinalSpillContext>,},- DrainingFinal {+ ProducingOutput {table: OrderedAggregateTable<FinalMarker>,},+ PreparingMergeInput {+ table: OrderedAggregateTable<FinalMarker>,+ spill_context: Box<OrderedFinalSpillContext>,+ },+ MergingSpills {+ stream: SendableRecordBatchStream,+ },Done,}@@ -64,6 +122,134 @@ type OrderedFinalAggregateStateTransition = ControlFlow<OrderedFinalAggregateState,>;+impl OrderedFinalSpillContext {+ fn new(+ agg: &AggregateExec,+ context: &Arc<TaskContext>,+ partition: usize,+ batch_size: usize,+ input_order_mode: &InputOrderMode,+ spill_schema: &SchemaRef,+ spill_metrics: SpillMetrics,+ ) -> Result<Self> {+ let group_schema = agg.group_by.group_schema(spill_schema)?;+ let output_ordering = agg.cache.output_ordering();+ let InputOrderMode::PartiallySorted(order_indices) = input_order_mode else {+ return internal_err!("Ordered final spill requires partially ordered input");+ };+ let spill_indices = order_indices.iter().copied().chain(+ (0..group_schema.fields().len()).filter(|idx| !order_indices.contains(idx)),+ );+ let spill_sort_exprs = spill_indices.map(|idx| {+ let field = group_schema.field(idx);+ let output_expr = Column::new(field.name(), idx);+ let sort_options = output_ordering+ .and_then(|ordering| ordering.get_sort_options(&output_expr))+ .unwrap_or_default();+ PhysicalSortExpr::new(Arc::new(output_expr), sort_options)+ });+ let Some(spill_expr) = LexOrdering::new(spill_sort_exprs) else {+ return internal_err!("Ordered final spill expression is empty");+ };++ let spill_manager = SpillManager::new(+ context.runtime_env(),+ spill_metrics,+ Arc::clone(spill_schema),+ )+ .with_compression_type(context.session_config().spill_compression());++ Ok(Self {+ agg: agg.clone(),+ context: Arc::clone(context),+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills: vec![],+ })+ }++ fn has_spills(&self) -> bool {+ !self.spills.is_empty()+ }++ /// Sorts and spills the aggregated groups. Memory reservation should be updated+ /// by the caller.+ ///+ /// Individual spill files are ordered by the `group by` keys.+ ///+ /// See [`OrderedFinalAggregateStream`] for spilling details.+ fn spill_table(+ &mut self,+ table: &mut OrderedAggregateTable<FinalMarker>,+ ) -> Result<()> {+ let Some(batch) = table.take_state_batch()? else {+ return Ok(());+ };++ let sorted_iter =+ IncrementalSortIterator::new(batch, self.spill_expr.clone(), self.batch_size);+ let spill_file = self+ .spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(+ sorted_iter,+ "OrderedFinalAggregateSpill",+ )?;++ let Some((file, max_record_batch_memory)) = spill_file else {+ return internal_err!("Ordered final aggregation produced an empty spill");+ };++ self.spills.push(SortedSpillFile {+ file,+ max_record_batch_memory,+ });++ Ok(())+ }++ /// Merges every sorted run and finalizes it through the fully ordered path.+ fn into_replay_stream(+ self,+ baseline_metrics: &BaselineMetrics,+ group_by_metrics: GroupByMetrics,+ reservation: MemoryReservation,+ ) -> Result<SendableRecordBatchStream> {+ let Self {+ agg,+ context,+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills,+ } = self;++ let spill_schema = Arc::clone(spill_manager.schema());+ let merged = StreamingMergeBuilder::new()+ .with_schema(spill_schema)+ .with_spill_manager(spill_manager)+ .with_sorted_spill_files(spills)+ .with_expressions(&spill_expr)+ .with_metrics(baseline_metrics.intermediate())+ .with_batch_size(batch_size)+ .with_reservation(reservation)+ .build()?;+ let replay = OrderedFinalAggregateStream::new_with_input_and_metrics(+ &agg,+ &context,+ partition,+ merged,+ &InputOrderMode::Sorted,+ baseline_metrics.clone(),+ group_by_metrics,+ None,+ )?;+ Ok(Box::pin(replay))+ }+}+impl OrderedFinalAggregateStream {pub fn new(agg: &AggregateExec,@@ -86,6 +272,35 @@ impl OrderedFinalAggregateStream {partition: usize,input: SendableRecordBatchStream,input_order_mode: &InputOrderMode,+ ) -> Result<Self> {+ let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);+ let group_by_metrics = GroupByMetrics::new(&agg.metrics, partition);+ let spill_metrics = SpillMetrics::new(&agg.metrics, partition);+ Self::new_with_input_and_metrics(+ agg,+ context,+ partition,+ input,+ input_order_mode,+ baseline_metrics,+ group_by_metrics,+ Some(spill_metrics),+ )+ }++ #[expect(+ clippy::too_many_arguments,+ reason = "keeps replay metric reuse explicit"+ )]+ fn new_with_input_and_metrics(+ agg: &AggregateExec,+ context: &Arc<TaskContext>,+ partition: usize,+ input: SendableRecordBatchStream,+ input_order_mode: &InputOrderMode,+ baseline_metrics: BaselineMetrics,+ group_by_metrics: GroupByMetrics,+ spill_metrics: Option<SpillMetrics>,) -> Result<Self> {debug_assert!(matches!(agg.mode,@@ -96,21 +311,37 @@ impl OrderedFinalAggregateStream {let schema = Arc::clone(&agg.schema);let input_schema = input.schema();let batch_size = context.session_config().batch_size();- let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);- // Preserve the existing aggregate metric surface for this plan node.- let _spill_metrics = SpillMetrics::new(&agg.metrics, partition);+ let can_spill = matches!(input_order_mode, InputOrderMode::PartiallySorted(_))+ && context.runtime_env().disk_manager.tmp_files_enabled();+ let spill_context = if can_spill {+ let Some(spill_metrics) = spill_metrics else {+ return internal_err!("Spillable ordered final stream requires metrics");+ };+ Some(Box::new(OrderedFinalSpillContext::new(+ agg,+ context,+ partition,+ batch_size,+ input_order_mode,+ &input_schema,+ spill_metrics,+ )?))+ } else {+ None+ };let table = OrderedAggregateTable::<FinalMarker>::new_with_input_order(agg,- partition,&input_schema,Arc::clone(&schema),batch_size,input_order_mode,+ group_by_metrics,)?;let reservation =MemoryConsumer::new(format!("OrderedFinalAggregateStream[{partition}]"))+ .with_can_spill(can_spill).register(context.memory_pool());Ok(Self {@@ -118,7 +349,10 @@ impl OrderedFinalAggregateStream {input,reservation,baseline_metrics,- state: Some(OrderedFinalAggregateState::ReadingInput { table }),+ state: Some(OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ }),})}@@ -127,6 +361,27 @@ impl OrderedFinalAggregateStream {self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));}+ fn break_with_internal_err(message: &str) -> OrderedFinalAggregateStateTransition {+ ControlFlow::Break((+ Poll::Ready(Some(internal_err!("{message}"))),+ OrderedFinalAggregateState::Done,+ ))+ }++ /// Reserve memory for the current aggregate table.+ fn reservation_size_for_table(+ table: &OrderedAggregateTable<FinalMarker>,+ spill_context: Option<&OrderedFinalSpillContext>,+ ) -> usize {+ let table_size = table.memory_size();+ if spill_context.is_some() {+ // See `OrderedFinalAggregateStream` comments for how is it estimated+ table_size.saturating_add(table.num_groups().saturating_mul(size_of::<u32>()))+ } else {+ table_size+ }+ }+/// Consumes one ordered partial-state input batch, then immediately emits/// finalized groups if the ordering proves any group is ready.///@@ -138,15 +393,23 @@ impl OrderedFinalAggregateStream {cx: &mut Context<'_>,original_state: OrderedFinalAggregateState,) -> OrderedFinalAggregateStateTransition {- let OrderedFinalAggregateState::ReadingInput { mut table } = original_state+ let OrderedFinalAggregateState::ReadingInput {+ mut table,+ spill_context,+ } = original_stateelse {- unreachable!("expected reading input state")+ return Self::break_with_internal_err(+ "Ordered final aggregate stream expected ReadingInput state",+ );};match self.input.poll_next_unpin(cx) {Poll::Pending => ControlFlow::Break((Poll::Pending,- OrderedFinalAggregateState::ReadingInput { table },+ OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ },)),Poll::Ready(Some(Ok(batch))) => {let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();@@ -157,21 +420,86 @@ impl OrderedFinalAggregateStream {if let Err(e) = result {return ControlFlow::Break((Poll::Ready(Some(Err(e))),- OrderedFinalAggregateState::ReadingInput { table },+ OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ },));}+ // Check memory reservation, and potentially spill.let timer = elapsed_compute.timer();- let result = table.next_output_batch();+ let resize_result =+ self.reservation+ .try_resize(Self::reservation_size_for_table(+ &table,+ spill_context.as_deref(),+ ));timer.done();+ match resize_result {+ Ok(()) => {}+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {+ let Some(spill_context) = spill_context else {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ ));+ };+ if table.is_empty() {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ ));+ }+ return ControlFlow::Continue(+ OrderedFinalAggregateState::Spilling {+ table,+ spill_context,+ },+ );+ }+ Err(e) => {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ ));+ }+ }++ let result = if spill_context+ .as_ref()+ .is_some_and(|spill_context| spill_context.has_spills())+ {+ // Once one incomplete run is spilled, every remaining state+ // must participate in replay so no group is finalized twice.+ Ok(None)+ } else {+ let timer = elapsed_compute.timer();+ let result = table.next_output_batch();+ timer.done();+ result+ };match result {// Some finalized groups can be emitted. Yield them, then// continue aggregating input in the current state.Ok(Some(batch)) => {- let next_state =- OrderedFinalAggregateState::ReadingInput { table };- self.resize_reservation_for_state(&next_state);+ if let Err(e) =+ self.reservation+ .try_resize(Self::reservation_size_for_table(+ &table,+ spill_context.as_deref(),+ ))+ {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ ));+ }+ let next_state = OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ };ControlFlow::Break((Poll::Ready(Some(Ok(@@ -180,36 +508,193 @@ impl OrderedFinalAggregateStream {next_state,))}+ // Can't do early emit, continue aggregating.Ok(None) => {- // Ordered variant doesn't support memory-limited- // execution, so it errors when memory reservation fails.- if let Err(e) = self.reservation.try_resize(table.memory_size()) {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- OrderedFinalAggregateState::ReadingInput { table },- ));- }-- // Can't do early emit, continue aggregating.ControlFlow::Continue(OrderedFinalAggregateState::ReadingInput {table,+ spill_context,})}Err(e) => ControlFlow::Break((Poll::Ready(Some(Err(e))),- OrderedFinalAggregateState::ReadingInput { table },+ OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ },)),}}Poll::Ready(Some(Err(e))) => ControlFlow::Break((Poll::Ready(Some(Err(e))),- OrderedFinalAggregateState::ReadingInput { table },+ OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context,+ },)),Poll::Ready(None) => {self.close_input();- table.input_done();- ControlFlow::Continue(OrderedFinalAggregateState::DrainingFinal { table })+ match spill_context {+ Some(spill_context) if spill_context.has_spills() => {+ ControlFlow::Continue(+ OrderedFinalAggregateState::PreparingMergeInput {+ table,+ spill_context,+ },+ )+ }+ _ => {+ table.input_done();+ ControlFlow::Continue(+ OrderedFinalAggregateState::ProducingOutput { table },+ )+ }+ }+ }+ }+ }++ /// Sorts and spills one complete in-memory state run, then resumes input.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_spilling(+ &mut self,+ original_state: OrderedFinalAggregateState,+ ) -> OrderedFinalAggregateStateTransition {+ let OrderedFinalAggregateState::Spilling {+ mut table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered final aggregate stream expected Spilling state",+ );+ };++ // Sanity check: it's impossible to OOM when the table is empty+ if table.is_empty() {+ return ControlFlow::Break((+ Poll::Ready(Some(internal_err!(+ "Ordered final aggregation entered Spilling with an empty table"+ ))),+ OrderedFinalAggregateState::Done,+ ));+ }++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let mut result = spill_context.spill_table(&mut table);++ // Spilling shrinks the aggregate table and releases its accumulated+ // memory. Update the reservation accordingly.+ if let Err(e) = self.reservation.try_resize(table.memory_size()) {+ result =+ Err(e.context("Decreasing allocation after spilling should succeed"));+ }++ timer.done();++ match result {+ // Finished spilling the aggregate table, continue aggregating from input+ Ok(()) => ControlFlow::Continue(OrderedFinalAggregateState::ReadingInput {+ table,+ spill_context: Some(spill_context),+ }),+ Err(e) => ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ )),+ }+ }++ /// 1. Spills the last in-memory run.+ /// 2. Constructs a globally ordered input stream by applying a sort-preserving+ /// merge to all spills.+ /// 3. Constructs a replay stream: an ordered aggregate stream over the fully+ /// ordered input constructed from the spills.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_preparing_merge_input(+ &mut self,+ original_state: OrderedFinalAggregateState,+ ) -> OrderedFinalAggregateStateTransition {+ let OrderedFinalAggregateState::PreparingMergeInput {+ mut table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered final aggregate stream expected PreparingMergeInput state",+ );+ };++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let replay = match spill_context.spill_table(&mut table) {+ Ok(()) => {+ let group_by_metrics = table.group_by_metrics();+ drop(table);+ match self.reservation.try_resize(0) {+ Ok(()) => (*spill_context).into_replay_stream(+ &self.baseline_metrics,+ group_by_metrics,+ self.reservation.new_empty(),+ ),+ Err(e) => Err(e),+ }+ }+ Err(e) => Err(e),+ };+ timer.done();++ match replay {+ Ok(stream) => {+ ControlFlow::Continue(OrderedFinalAggregateState::MergingSpills {+ stream,+ })}+ Err(e) => ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ )),+ }+ }++ /// Forwards output from the fully ordered stream that consumes the merged+ /// spill runs.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_merging_spills(+ &mut self,+ cx: &mut Context<'_>,+ original_state: OrderedFinalAggregateState,+ ) -> OrderedFinalAggregateStateTransition {+ let OrderedFinalAggregateState::MergingSpills { mut stream } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered final aggregate stream expected MergingSpills state",+ );+ };++ match stream.poll_next_unpin(cx) {+ Poll::Pending => ControlFlow::Break((+ Poll::Pending,+ OrderedFinalAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Ok(batch))) => ControlFlow::Break((+ Poll::Ready(Some(Ok(batch))),+ OrderedFinalAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Err(e))) => ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ )),+ Poll::Ready(None) => ControlFlow::Continue(OrderedFinalAggregateState::Done),}}@@ -221,12 +706,14 @@ impl OrderedFinalAggregateStream {/// See comments at `poll_next()` for details.////// Returns the next operator state with control flow decision.- fn handle_draining_final(+ fn handle_producing_output(&mut self,original_state: OrderedFinalAggregateState,) -> OrderedFinalAggregateStateTransition {- let OrderedFinalAggregateState::DrainingFinal { table } = original_state else {- unreachable!("expected draining final state")+ let OrderedFinalAggregateState::ProducingOutput { table } = original_state else {+ return Self::break_with_internal_err(+ "Ordered final aggregate stream expected ProducingOutput state",+ );};let mut table = table;@@ -238,11 +725,23 @@ impl OrderedFinalAggregateStream {match result {Ok(Some(batch)) => {let next_state = if table.is_empty() {+ drop(table);+ if let Err(e) = self.reservation.try_resize(0) {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::Done,+ ));+ }OrderedFinalAggregateState::Done} else {- OrderedFinalAggregateState::DrainingFinal { table }+ if let Err(e) = self.reservation.try_resize(table.memory_size()) {+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedFinalAggregateState::ProducingOutput { table },+ ));+ }+ OrderedFinalAggregateState::ProducingOutput { table }};- self.resize_reservation_for_state(&next_state);ControlFlow::Break((Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),@@ -251,24 +750,18 @@ impl OrderedFinalAggregateStream {}Err(e) => ControlFlow::Break((Poll::Ready(Some(Err(e))),- OrderedFinalAggregateState::DrainingFinal { table },+ OrderedFinalAggregateState::ProducingOutput { table },)),Ok(None) => {+ drop(table);let next_state = OrderedFinalAggregateState::Done;- self.resize_reservation_for_state(&next_state);+ if let Err(e) = self.reservation.try_resize(0) {+ return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state));+ }ControlFlow::Continue(next_state)}}}-- fn resize_reservation_for_state(&mut self, state: &OrderedFinalAggregateState) {- let new_size = match state {- OrderedFinalAggregateState::ReadingInput { table }- | OrderedFinalAggregateState::DrainingFinal { table } => table.memory_size(),- OrderedFinalAggregateState::Done => 0,- };- let _ = self.reservation.try_resize(new_size);- }}impl Stream for OrderedFinalAggregateStream {@@ -288,15 +781,37 @@ impl Stream for OrderedFinalAggregateStream {////// ReadingInput/// -> ReadingInput- /// Merge one input batch. If the ordering proves some groups are- /// complete, yield one final aggregate batch immediately, then continue- /// reading input. Otherwise continue directly with the next input batch.- /// -> DrainingFinal- /// Input was exhausted. Mark the table input as done so every remaining- /// group is safe to emit.+ /// Merge one input batch. If it fits in memory, optionally yield groups+ /// proven complete by the input ordering, then read the next batch.+ /// -> Spilling+ /// The table cannot reserve enough memory. Move all current states into+ /// one fully group-key-sorted spill run.+ /// -> ProducingOutput+ /// Input was exhausted without spilling. Mark every remaining group as+ /// complete and produce its final result.+ /// -> PreparingMergeInput+ /// Input was exhausted after spilling. Spill the last in-memory run and+ /// construct the ordered input used to merge all spill files.+ ///+ /// Spilling+ /// -> ReadingInput+ /// One sorted run was written; resume reading the original input.+ ///+ /// PreparingMergeInput+ /// Spill the final in-memory run and build the input ordered replay stream.+ /// -> MergingSpills+ /// The final run was spilled and the ordered replay stream was built.///- /// DrainingFinal- /// -> DrainingFinal+ /// MergingSpills+ /// Aggregate the merged spill runs and emit final results.+ /// -> MergingSpills+ /// Forward one result batch from the fully ordered replay stream that+ /// consumes the sort-preserving merge.+ /// -> Done+ /// The merged spill input was fully aggregated.+ ///+ /// ProducingOutput+ /// -> ProducingOutput/// One remaining final aggregate batch was yielded; repeat to continue/// draining the table./// -> Done@@ -319,8 +834,17 @@ impl Stream for OrderedFinalAggregateStream {state @ OrderedFinalAggregateState::ReadingInput { .. } => {self.handle_reading_input(cx, state)}- state @ OrderedFinalAggregateState::DrainingFinal { .. } => {- self.handle_draining_final(state)+ state @ OrderedFinalAggregateState::Spilling { .. } => {+ self.handle_spilling(state)+ }+ state @ OrderedFinalAggregateState::PreparingMergeInput { .. } => {+ self.handle_preparing_merge_input(state)+ }+ state @ OrderedFinalAggregateState::MergingSpills { .. } => {+ self.handle_merging_spills(cx, state)+ }+ state @ OrderedFinalAggregateState::ProducingOutput { .. } => {+ self.handle_producing_output(state)}state @ OrderedFinalAggregateState::Done => {let _ = self.reservation.try_resize(0);@@ -334,6 +858,15 @@ impl Stream for OrderedFinalAggregateStream {self.state = Some(next_state);continue;}+ ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {+ // Errors are terminal: discard all operator state and release+ // its upstream input and memory reservation before returning.+ drop(next_state);+ self.close_input();+ self.reservation.free();+ self.state = Some(OrderedFinalAggregateState::Done);+ return Poll::Ready(Some(Err(e)));+ }ControlFlow::Break((poll, next_state)) => {self.state = Some(next_state);return poll;diff --git a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rsindex b4b7fa073a..73d15a8278 100644--- a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs@@ -23,7 +23,7 @@ use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::Result;+use datafusion_common::{DataFusionError, Result};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};use futures::stream::{Stream, StreamExt};@@ -31,6 +31,7 @@ use futures::stream::{Stream, StreamExt};use super::AggregateExec;use super::aggregate_hash_table::{OrderedAggregateTable, PartialMarker};use crate::aggregates::AggregateMode;+use crate::aggregates::order::GroupOrdering;use crate::metrics::{BaselineMetrics, MetricBuilder, RecordOutput, SpillMetrics};use crate::stream::EmptyRecordBatchStream;use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream, metrics};@@ -69,6 +70,41 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream, metric/// `k = 100`, it is safe to emit all groups with keys less than 100 because the/// input is ordered.///+/// # Memory Pressure and Spilling+///+/// ## Fully ordered case+///+/// If the input is ordered by every group key, for example:+///+/// - Input order: `a, b`+/// - `GROUP BY`: `a, b`+///+/// Completed groups can be emitted as soon as the next group is observed. Thus,+/// only the current group remains active after completed groups are emitted, and+/// memory usage does not grow with the total number of groups.+///+/// If a memory reservation nevertheless fails, the stream returns the error+/// directly, indicating an unexpected behavior.+///+/// ## Partially ordered case+///+/// If the input is ordered by only a subset of the group keys, for example:+///+/// - Input order: `a`+/// - `GROUP BY`: `a, b`+///+/// If one `a` value contains many distinct `b` values, the table may accumulate+/// enough groups to exceed the memory limit.+///+/// - `OrderedPartialAggregateStream`: On reservation failure, it emits all current+/// intermediate states downstream and resets the table. The final stage can+/// merge repeated `(a, b)` state rows, so no disk spill is required.+/// - `OrderedFinalAggregateStream`: It cannot emit incomplete final results. On+/// reservation failure, it sorts the current intermediate states by the complete+/// group key and spills them as one run. After the input ends, it spills any+/// remaining states, performs a sort-preserving merge of all runs, and feeds the+/// merged input into a fully ordered final aggregate stream.+////// ## Implementation Note////// This is intentionally kept simple and closely maps to@@ -76,7 +112,6 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream, metric////// See issue for details: <https://github.com/apache/datafusion/issues/22710>///-/// More applicable optimizations are left to future work.pub(crate) struct OrderedPartialAggregateStream {schema: SchemaRef,input: SendableRecordBatchStream,@@ -131,6 +166,10 @@ impl OrderedPartialAggregateStream {)?;let reservation =MemoryConsumer::new(format!("OrderedPartialAggregateStream[{partition}]"))+ .with_can_spill(matches!(+ table.group_ordering(),+ GroupOrdering::Partial(_)+ )).register(context.memory_pool());Ok(Self {@@ -185,6 +224,28 @@ impl OrderedPartialAggregateStream {));}+ // Check memory reservation. See function comments for details.+ match self.resize_or_take_state_batch(&mut table) {+ Ok(Some(batch)) => {+ self.reduction_factor.add_part(batch.num_rows());+ return ControlFlow::Break((+ Poll::Ready(Some(Ok(+ batch.record_output(&self.baseline_metrics)+ ))),+ OrderedPartialAggregateState::ReadingInput { table },+ ));+ }+ Ok(None) => {}+ Err(e) => {+ self.close_input();+ self.reservation.free();+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedPartialAggregateState::Done,+ ));+ }+ }+let timer = elapsed_compute.timer();let result = table.next_output_batch();timer.done();@@ -195,9 +256,16 @@ impl OrderedPartialAggregateStream {// current state)Ok(Some(batch)) => {self.reduction_factor.add_part(batch.num_rows());+ if let Err(e) = self.reservation.try_resize(table.memory_size()) {+ self.close_input();+ self.reservation.free();+ return ControlFlow::Break((+ Poll::Ready(Some(Err(e))),+ OrderedPartialAggregateState::Done,+ ));+ }let next_state =OrderedPartialAggregateState::ReadingInput { table };- self.resize_reservation_for_state(&next_state);ControlFlow::Break((Poll::Ready(Some(Ok(@@ -206,21 +274,10 @@ impl OrderedPartialAggregateStream {next_state,))}- Ok(None) => {- // Ordered variant don't support memory-limited execution,- // it have to error when OOM- if let Err(e) = self.reservation.try_resize(table.memory_size()) {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- OrderedPartialAggregateState::ReadingInput { table },- ));- }-- // Can't do early emit, continue aggregating.- ControlFlow::Continue(- OrderedPartialAggregateState::ReadingInput { table },- )- }+ // Can't do early emit, continue aggregating.+ Ok(None) => ControlFlow::Continue(+ OrderedPartialAggregateState::ReadingInput { table },+ ),Err(e) => ControlFlow::Break((Poll::Ready(Some(Err(e))),OrderedPartialAggregateState::ReadingInput { table },@@ -242,6 +299,42 @@ impl OrderedPartialAggregateStream {}}+ /// Update the memory reservation, and:+ /// - If memory reservation succeed, returns `Ok(None)`+ /// - If memory reservation failed,+ /// - If input is partially ordered, materialize all the output, and+ /// directly send them to the final aggregation stage.+ /// Returns `Ok(Some(batch))`+ /// - If input is fully ordered, directly return error. It's not+ /// expected to use more than constant memory.+ /// Returns `Err(..)`+ ///+ /// # Implementation Note+ /// Incrementally output it after the blocked state management is ready, keep+ /// it simple for now.+ ///+ /// Issue: <https://github.com/apache/datafusion/issues/7065>+ fn resize_or_take_state_batch(+ &mut self,+ table: &mut OrderedAggregateTable<PartialMarker>,+ ) -> Result<Option<RecordBatch>> {+ let oom = match self.reservation.try_resize(table.memory_size()) {+ Ok(()) => return Ok(None),+ Err(e @ DataFusionError::ResourcesExhausted(_)) => e,+ Err(e) => return Err(e),+ };++ if matches!(table.group_ordering(), GroupOrdering::Full(_)) {+ return Err(oom);+ }++ let Some(batch) = table.take_state_batch()? else {+ return Err(oom);+ };+ self.reservation.try_resize(table.memory_size())?;+ Ok(Some(batch))+ }+/// Emits one batch after input is exhausted.////// `table.input_done()` has already made every remaining group safe to emit,@@ -272,7 +365,9 @@ impl OrderedPartialAggregateStream {} else {OrderedPartialAggregateState::DrainingFinal { table }};- self.resize_reservation_for_state(&next_state);+ if let Err(e) = self.resize_reservation_for_state(&next_state) {+ return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state));+ }ControlFlow::Break((Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),@@ -285,13 +380,18 @@ impl OrderedPartialAggregateStream {)),Ok(None) => {let next_state = OrderedPartialAggregateState::Done;- self.resize_reservation_for_state(&next_state);+ if let Err(e) = self.resize_reservation_for_state(&next_state) {+ return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state));+ }ControlFlow::Continue(next_state)}}}- fn resize_reservation_for_state(&mut self, state: &OrderedPartialAggregateState) {+ fn resize_reservation_for_state(+ &mut self,+ state: &OrderedPartialAggregateState,+ ) -> Result<()> {let new_size = match state {OrderedPartialAggregateState::ReadingInput { table }| OrderedPartialAggregateState::DrainingFinal { table } => {@@ -299,7 +399,7 @@ impl OrderedPartialAggregateStream {}OrderedPartialAggregateState::Done => 0,};- let _ = self.reservation.try_resize(new_size);+ self.reservation.try_resize(new_size)}}diff --git a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.sltnew file mode 100644index 0000000000..a4d492ab82--- /dev/null+++ b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt@@ -0,0 +1,201 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# End-to-end tests for ordered aggregation under finite memory.++# Result set more than 100 lines will be hashed+hash-threshold 100++statement ok+SET datafusion.execution.target_partitions = 2++statement ok+SET datafusion.execution.batch_size = 128++statement ok+SET datafusion.optimizer.repartition_aggregations = true++statement ok+SET datafusion.optimizer.prefer_existing_sort = true++statement ok+SET datafusion.execution.enable_migration_aggregate = true++statement ok+SET datafusion.runtime.memory_limit = '1M'++# ==================================================================================+# Input is fully ordered by group keys (input order by (a,b), query is 'group by a,b')+# ==================================================================================++# Fully ordered input uses the ordered partial and final streams without spill.+query TT+EXPLAIN ANALYZE+SELECT v1, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY v1+----+Plan with Metrics+01)AggregateExec: mode=FinalPartitioned,<slt:ignore>ordering_mode=Sorted, metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+02)--RepartitionExec:<slt:ignore>preserve_order=true<slt:ignore>+03)----AggregateExec: mode=Partial,<slt:ignore>ordering_mode=Sorted, metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+<slt:ignore>++query II rowsort+SELECT v1, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY v1+----+40002 values hashing to 34c2b23730596cbd2489ed4a627c17d7++# The same fully ordered query cannot spill and reports OOM under tighter memory.+statement ok+SET datafusion.runtime.memory_limit = '1K'++query error Resources exhausted+SELECT v1, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY v1++# ==================================================================================+# Input is partially ordered by group keys (input order by (a), query is 'group by a,b')+#+# Try different memory limits, ensure result is the same, but spill count differ++# HACK: check `spilled_bytes=x KB` to ensure it has spilled. If it has not spilled,+# the it shows `spilled_bytes = 0B`. Should better check spill_count, but it's not+# stable due to ordered hash repartition, and `sqllogictest` don't support regex.+# ==================================================================================++statement ok+SET datafusion.runtime.memory_limit = '2M'++statement ok+SET datafusion.optimizer.enable_round_robin_repartition = false++# Round 1: The input is partially ordered and does not spill with a 2 MB limit.+query TT+EXPLAIN ANALYZE+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+Plan with Metrics+01)AggregateExec: mode=FinalPartitioned,<slt:ignore>aggr=[sum(t1.v1 * Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+02)--RepartitionExec:<slt:ignore>input_partitions=1, maintains_sort_order=true<slt:ignore>+03)----AggregateExec: mode=Partial,<slt:ignore>ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+<slt:ignore>++# All rounds should have the same result hash+query III rowsort+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+45000 values hashing to e6ece4b4b86e6152a1c785e90ab5ba12++# Round 2: The same query spills five times with a 600 KB limit.+statement ok+SET datafusion.runtime.memory_limit = '600K'++query TT+EXPLAIN ANALYZE+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+Plan with Metrics+01)AggregateExec: mode=FinalPartitioned,<slt:ignore>aggr=[sum(t1.v1 * Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spilled_bytes=<slt:ignore> KB,<slt:ignore>]+02)--RepartitionExec:<slt:ignore>input_partitions=1, maintains_sort_order=true<slt:ignore>+03)----AggregateExec: mode=Partial,<slt:ignore>ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+<slt:ignore>++# All rounds should have the same result hash+query III rowsort+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+45000 values hashing to e6ece4b4b86e6152a1c785e90ab5ba12++# Round 3: The same query spills six times with a 500 KB limit.+statement ok+SET datafusion.runtime.memory_limit = '500K'++query TT+EXPLAIN ANALYZE+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+Plan with Metrics+01)AggregateExec: mode=FinalPartitioned,<slt:ignore>aggr=[sum(t1.v1 * Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spilled_bytes=<slt:ignore> KB,<slt:ignore>]+02)--RepartitionExec:<slt:ignore>input_partitions=1, maintains_sort_order=true<slt:ignore>+03)----AggregateExec: mode=Partial,<slt:ignore>ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+<slt:ignore>++# All rounds should have the same result hash+query III rowsort+SELECT round(v1, -4), v1 % 5000, sum(v1 * 2)+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+45000 values hashing to e6ece4b4b86e6152a1c785e90ab5ba12++# Exercise the same spill path with a variable-width string aggregate state in+# the spilled payload. Keep one partial input partition so memory pressure is on+# the ordered aggregate rather than a repartition merge.+statement ok+SET datafusion.runtime.memory_limit = '600K'++# Ensures final aggregate has spill_count > 0+query TT+EXPLAIN ANALYZE+SELECT round(v1, -4), v1 % 5000,+ sum(v1 * 2), min(CAST(v1 % 2 AS VARCHAR))+FROM generate_series(20000) AS t1(v1)+GROUP BY round(v1, -4), v1 % 5000+----+Plan with Metrics+01)AggregateExec: mode=FinalPartitioned,<slt:ignore>aggr=[sum(t1.v1 * Int64(2)), min(t1.v1 % Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spilled_bytes=<slt:ignore>KB,<slt:ignore>]+02)--RepartitionExec:<slt:ignore>input_partitions=1, maintains_sort_order=true<slt:ignore>+03)----AggregateExec: mode=Partial,<slt:ignore>aggr=[sum(t1.v1 * Int64(2)), min(t1.v1 % Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[<slt:ignore>spill_count=0,<slt:ignore>]+<slt:ignore>++statement ok+RESET datafusion.runtime.memory_limit++statement ok+RESET datafusion.optimizer.enable_round_robin_repartition++statement ok+RESET datafusion.execution.enable_migration_aggregate++statement ok+RESET datafusion.optimizer.prefer_existing_sort++statement ok+RESET datafusion.optimizer.repartition_aggregations++statement ok+RESET datafusion.execution.batch_size++statement ok+SET datafusion.execution.target_partitions = 4++statement ok+RESET datafusion.catalog.create_default_catalog_and_schema