Apache DataFusion SQL Query Engine
refactor(hash-aggr): Support spilling for single mode aggregation (#23965)
Yongting You212d6137f11c
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex 42014f336f..91e9d6555c 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -82,6 +82,10 @@ pub(in crate::aggregates) struct AggregateHashTable<AggrMode> {/// Output schema: group columns followed by aggregate state or final values.pub(super) output_schema: SchemaRef,+ /// Intermediate-state schema used when memory pressure requires the table+ /// to spill its current state.+ pub(super) state_schema: SchemaRef,+/// Maximum rows per emitted output batch, from config `batch_size`.pub(super) batch_size: usize,@@ -97,6 +101,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {agg: &AggregateExec,partition: usize,output_schema: SchemaRef,+ state_schema: SchemaRef,batch_size: usize,filters: Vec<Option<Arc<dyn PhysicalExpr>>>,) -> Result<Self> {@@ -133,6 +138,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {group_by_metrics: GroupByMetrics::new(&agg.metrics, partition),input_schema,output_schema,+ state_schema,batch_size,state: AggregateHashTableState::Building(AggregateHashTableBuffer {group_by: Arc::clone(&agg.group_by),@@ -282,11 +288,48 @@ impl<AggrMode> AggregateHashTable<AggrMode> {}}+ pub(in crate::aggregates) fn group_by_metrics(&self) -> &GroupByMetrics {+ &self.group_by_metrics+ }+/// Returns the number of distinct groups accumulated so far.pub(in crate::aggregates) fn building_group_count(&self) -> usize {self.state.building().group_values.len()}+ /// Takes every intermediate aggregate state and resets the table so it can+ /// continue accumulating raw input.+ ///+ /// Unlike normal single aggregation output, this materializes intermediate+ /// states rather than final values. The states can therefore be merged after+ /// spilling without finalizing the same group more than once.+ pub(in crate::aggregates) fn take_state_batch(+ &mut self,+ ) -> Result<Option<RecordBatch>> {+ let state_schema = Arc::clone(&self.state_schema);+ let state = self.state.building_mut();+ if state.group_values.is_empty() {+ return Ok(None);+ }++ let mut output = state.group_values.emit(EmitTo::All)?;+ for acc in &mut state.accumulators {+ output.extend(acc.state(EmitTo::All)?);+ }++ let batch = RecordBatch::try_new(state_schema, output)?;+ debug_assert!(batch.num_rows() > 0);++ // `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the+ // key/index buffers too so the memory reservation can be released+ // before the batch is sorted for spilling.+ state.group_values.clear_shrink(0);+ state.batch_group_indices.clear();+ state.batch_group_indices.shrink_to_fit();++ Ok(Some(batch))+ }+pub(in crate::aggregates) fn is_building(&self) -> bool {matches!(self.state, AggregateHashTableState::Building(_))}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rsindex 522cc9066b..b80e15d7f8 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;+use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;@@ -41,6 +43,7 @@ impl AggregateHashTable<FinalMarker> {agg,partition,output_schema,+ Arc::clone(&agg.input().schema()),batch_size,vec![None; agg.aggr_expr.len()],)diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rsindex d8e92c5928..4dfd6a74d1 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs@@ -15,6 +15,8 @@// specific language governing permissions and limitations// under the License.+use std::sync::Arc;+use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;@@ -34,6 +36,7 @@ impl AggregateHashTable<PartialReduceMarker> {Self::new_with_filters(agg,partition,+ Arc::clone(&output_schema),output_schema,batch_size,vec![None; agg.aggr_expr.len()],diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rsindex 4bcacb49af..a64fd32536 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs@@ -50,6 +50,7 @@ impl AggregateHashTable<PartialMarker> {Self::new_with_filters(agg,partition,+ Arc::clone(&output_schema),output_schema,batch_size,agg.filter_expr.iter().cloned().collect(),@@ -87,6 +88,7 @@ impl AggregateHashTable<PartialMarker> {group_by_metrics: self.group_by_metrics.clone(),input_schema: Arc::clone(&self.input_schema),output_schema: Arc::clone(&self.output_schema),+ state_schema: Arc::clone(&self.state_schema),batch_size: self.batch_size,state: AggregateHashTableState::Building(AggregateHashTableBuffer {group_by: Arc::clone(&state.group_by),diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/single_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/single_table.rsindex 5dcb735d08..56d601c793 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/single_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/single_table.rs@@ -35,12 +35,14 @@ impl AggregateHashTable<SingleMarker> {agg: &AggregateExec,partition: usize,output_schema: SchemaRef,+ state_schema: SchemaRef,batch_size: usize,) -> Result<Self> {Self::new_with_filters(agg,partition,output_schema,+ state_schema,batch_size,agg.filter_expr.iter().cloned().collect(),)diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 0523628ad7..33860d3f51 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1273,12 +1273,7 @@ impl AggregateExec {&& self.group_by.is_single()}- fn should_use_single_hash_stream(&self, context: &TaskContext) -> bool {- // TODO: implement memory-limited path and remove this limitation- if matches!(context.memory_pool().memory_limit(), MemoryLimit::Finite(_)) {- return false;- }-+ fn should_use_single_hash_stream(&self, _context: &TaskContext) -> bool {matches!(self.mode,AggregateMode::Single | AggregateMode::SinglePartitioned@@ -3709,7 +3704,7 @@ mod tests {let aggregates_v0: Vec<Arc<AggregateFunctionExpr>> =vec![Arc::new(test_median_agg_expr(Arc::clone(&input_schema))?)];- // use fast-path in `grouped_hash_stream.rs`.+ // Use the fast path in `single_stream.rs`.let aggregates_v2: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(AggregateExprBuilder::new(avg_udaf(), vec![col("b", &input_schema)?]).schema(Arc::clone(&input_schema))@@ -3742,7 +3737,7 @@ mod tests {assert!(matches!(stream, StreamType::GroupedHash(_)));}2 => {- assert!(matches!(stream, StreamType::GroupedHash(_)));+ assert!(matches!(stream, StreamType::SingleHash(_)));}_ => panic!("Unknown version: {version}"),}@@ -4077,15 +4072,14 @@ mod tests {Ok(())}- /// Spilling behavior is not implemented for single hash stream yet, so fall- /// back to the existing `GroupedHashAggregateStream`.+ /// Single hash aggregation supports finite memory.#[tokio::test]async fn single_aggregate_with_memory_limit_planning() -> Result<()> {let single = single_test_aggregate()?;let task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;let stream = single.execute_typed(0, &task_ctx)?;- assert!(matches!(stream, StreamType::GroupedHash(_)));+ assert!(matches!(stream, StreamType::SingleHash(_)));Ok(())}@@ -5605,9 +5599,11 @@ mod tests {Field::new("b", DataType::Float64, false),]));+ let group_keys = [2, 3, 4, 4].repeat(1_000);+ let values = [1.0, 2.0, 3.0, 4.0].repeat(1_000);let batches = vec![- create_record_batch(&schema, (vec![2, 3, 4, 4], vec![1.0, 2.0, 3.0, 4.0]))?,- create_record_batch(&schema, (vec![2, 3, 4, 4], vec![1.0, 2.0, 3.0, 4.0]))?,+ create_record_batch(&schema, (group_keys.clone(), values.clone()))?,+ create_record_batch(&schema, (group_keys, values))?,];let plan: Arc<dyn ExecutionPlan> =TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;@@ -5707,9 +5703,9 @@ mod tests {#[tokio::test]async fn test_aggregate_with_spill_if_necessary() -> Result<()> {// test with spill- run_test_with_spill_pool_if_necessary(2_000, true).await?;+ run_test_with_spill_pool_if_necessary(20_000, true).await?;// test without spill- run_test_with_spill_pool_if_necessary(20_000, false).await?;+ run_test_with_spill_pool_if_necessary(200_000, false).await?;Ok(())}@@ -6685,11 +6681,6 @@ mod tests {matches!(root, DataFusionError::ResourcesExhausted(_)),"Expected ResourcesExhausted, got: {root}",);- let msg = root.to_string();- assert!(- msg.contains("Failed to reserve memory for sort during spill"),- "Expected sort reservation error, got: {msg}",- );}}diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rsindex 071c1e9011..26f644d8b6 100644--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs@@ -97,6 +97,9 @@ struct OrderedFinalSpillContext {enum OrderedFinalAggregateState {ReadingInput {table: OrderedAggregateTable<FinalMarker>,+ /// None if either+ /// - Disk Manager doesn't enable temporary file creation+ /// - The group keys are fully ordered, it's expected to use bounded memoryspill_context: Option<Box<OrderedFinalSpillContext>>,},Spilling {@@ -292,7 +295,7 @@ impl OrderedFinalAggregateStream {clippy::too_many_arguments,reason = "keeps replay metric reuse explicit")]- fn new_with_input_and_metrics(+ pub(in crate::aggregates) fn new_with_input_and_metrics(agg: &AggregateExec,context: &Arc<TaskContext>,partition: usize,@@ -440,6 +443,8 @@ impl OrderedFinalAggregateStream {Ok(()) => {}Err(e @ DataFusionError::ResourcesExhausted(_)) => {let Some(spill_context) = spill_context else {+ // `None` means spilling is not supported, see comments+ // at `OrderedFinalAggregateState` for details.return ControlFlow::Break((Poll::Ready(Some(Err(e))),OrderedFinalAggregateState::Done,diff --git a/datafusion/physical-plan/src/aggregates/single_stream.rs b/datafusion/physical-plan/src/aggregates/single_stream.rsindex 886ffdd3a0..2917b960f6 100644--- a/datafusion/physical-plan/src/aggregates/single_stream.rs+++ b/datafusion/physical-plan/src/aggregates/single_stream.rs@@ -28,26 +28,42 @@ use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;-use datafusion_common::Result;+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr_common::sort_expr::LexOrdering;use futures::stream::{Stream, StreamExt};-use super::AggregateExec;use super::aggregate_hash_table::{AggregateHashTable, SingleMarker};-use crate::metrics::{BaselineMetrics, RecordOutput};+use super::group_values::GroupByMetrics;+use super::ordered_final_stream::OrderedFinalAggregateStream;+use super::{AggregateExec, create_schema};+use crate::aggregates::AggregateMode;+use crate::metrics::{BaselineMetrics, RecordOutput, SpillMetrics};+use crate::sorts::IncrementalSortIterator;+use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};+use crate::spill::spill_manager::SpillManager;use crate::stream::EmptyRecordBatchStream;use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};/// Hash aggregation can run the full logical aggregation in one operator. This/// stream implements the single stage for grouped hash aggregation.///+/// This aggregation variant is useful when:+/// - There is only one partition (config `target_partitions` is set to 1)+/// - When input is already partitioned (`t` is backed by Parquet files, that is range/hash+/// partitioned on the group keys), the single aggregation mode is the most efficient+/// approach to use.+////// # Example////// SELECT k, AVG(v) FROM t GROUP BY k;////// ## Plan/// AggregateExec(stage=single)+/// -- DataSourceExec(t)////// ## Single Stage Behavior/// Input: raw rows@@ -55,6 +71,19 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};////// This stream implements the complete aggregation without a partial/final/// split. It consumes raw input rows and emits final aggregate values.+///+/// # Spilling+///+/// During aggregation, group keys and states accumulate. If memory usage exceeds+/// the budget, spilling is triggered as follows:+/// 1. After aggregating a new input batch, if the memory reservation exceeds its+/// limit, spill all accumulated groups and states.+/// - Sort all groups by the group keys before spilling.+/// 2. Repeat until the input is exhausted.+/// 3. Perform a sort-preserving merge of all spill files and feed the merged output+/// into an ordered streaming aggregation, which ensures bounded memory usage and+/// evaluates the final result.+/// - [`OrderedFinalAggregateStream`] is reused for the streaming aggregation.pub(crate) struct SingleHashAggregateStream {/// Output schema: group columns followed by final aggregate value columns.schema: SchemaRef,@@ -65,7 +94,7 @@ pub(crate) struct SingleHashAggregateStream {/// Execution metrics shared with the aggregate plan node.baseline_metrics: BaselineMetrics,- /// Memory reservation for group keys and accumulators.+ /// Memory reservation for group keys, accumulators, and spill sorting.reservation: MemoryReservation,/// Tracks the high-level stream lifecycle. The hash table owns the lower-level@@ -73,17 +102,60 @@ pub(crate) struct SingleHashAggregateStream {state: Option<SingleHashAggregateState>,}-/// States for single hash aggregation processing.-// The typestate pattern mirrors the final stream and keeps the input/output-// semantics explicit for this mode.+/// Spill configuration and accumulated runs for single hash aggregation.+///+/// Each spill event drains all currently buffered groups, sorts their intermediate+/// states by the full group key, and writes them to one spill file. All files are+/// merged and replayed after the original input ends.+struct SingleSpillContext {+ /// Aggregate configuration used to construct the final replay stream.+ ///+ /// Spilled rows already contain evaluated group keys and intermediate+ /// aggregate states. Replay must therefore use final aggregation semantics+ /// and column-based group expressions rather than evaluating the raw input+ /// expressions a second time. After the spill files are merged into ordered+ /// input, this configuration is used to construct an+ /// [`OrderedFinalAggregateStream`], and perform the final evaluation step.+ final_agg: AggregateExec,+ /// Task context.+ context: Arc<TaskContext>,+ /// Original partition index.+ partition: usize,+ /// Target batch size from configuration.+ batch_size: usize,+ /// Full group-key ordering kept by every spill file and the merged input.+ spill_expr: LexOrdering,+ /// Spill I/O and metrics manager.+ spill_manager: SpillManager,+ /// Spill runs waiting to be merged, they're all sorted by full group-by keys.+ spills: Vec<SortedSpillFile>,+}++/// See comments at `poll_next()` for details.enum SingleHashAggregateState {ReadingInput {hash_table: AggregateHashTable<SingleMarker>,+ spill_context: Option<Box<SingleSpillContext>>,+ },+ Spilling {+ hash_table: AggregateHashTable<SingleMarker>,+ spill_context: Box<SingleSpillContext>,},ProducingOutput {hash_table: AggregateHashTable<SingleMarker>,},+ PreparingMergeInput {+ hash_table: AggregateHashTable<SingleMarker>,+ spill_context: Box<SingleSpillContext>,+ },+ MergingSpills {+ stream: SendableRecordBatchStream,+ },Done,+ /// Sentinel state to use when returning error from any other states, because:+ /// - It explicitly releases state-owned resources immediately+ /// - More defensive against accidentally resuming execution after error+ Error,}type SingleHashAggregatePoll = Poll<Option<Result<RecordBatch>>>;@@ -92,42 +164,143 @@ type SingleHashAggregateStateTransition = ControlFlow<SingleHashAggregateState,>;-impl SingleHashAggregateState {- fn hash_table(&self) -> &AggregateHashTable<SingleMarker> {- match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table+impl SingleSpillContext {+ fn new(+ agg: &AggregateExec,+ context: &Arc<TaskContext>,+ partition: usize,+ batch_size: usize,+ spill_schema: &SchemaRef,+ spill_metrics: SpillMetrics,+ ) -> Result<Self> {+ let group_schema = agg.group_by.group_schema(&agg.input().schema())?;+ let output_ordering = agg.cache.output_ordering();+ let spill_sort_exprs =+ group_schema+ .fields()+ .iter()+ .enumerate()+ .map(|(idx, field)| {+ let output_expr = Column::new(field.name(), idx);+ let sort_options = output_ordering+ .and_then(|ordering| ordering.get_sort_options(&output_expr))+ .unwrap_or_default();+ PhysicalSortExpr::new(Arc::new(output_expr), sort_options)+ });+ let Some(spill_expr) = LexOrdering::new(spill_sort_exprs) else {+ return internal_err!("Single hash aggregate spill expression is empty");+ };++ let spill_manager = SpillManager::new(+ context.runtime_env(),+ spill_metrics,+ Arc::clone(spill_schema),+ )+ .with_compression_type(context.session_config().spill_compression());++ // See `SingleSpillContext::final_agg` comments for `final_agg`'s usage+ let mut final_agg = agg.clone();+ final_agg.mode = match agg.mode {+ AggregateMode::Single => AggregateMode::Final,+ AggregateMode::SinglePartitioned => AggregateMode::FinalPartitioned,+ mode => {+ return internal_err!(+ "Single hash aggregate spill cannot replay aggregate mode {mode:?}"+ );}- Self::Done => unreachable!("Done state does not hold a hash table"),- }- }+ };+ final_agg.group_by = Arc::new(agg.group_by.as_final());+ final_agg.input_order_mode = InputOrderMode::Sorted;- fn hash_table_mut(&mut self) -> &mut AggregateHashTable<SingleMarker> {- match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table- }- Self::Done => unreachable!("Done state does not hold a hash table"),- }+ Ok(Self {+ final_agg,+ context: Arc::clone(context),+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills: vec![],+ })}- fn into_hash_table(self) -> AggregateHashTable<SingleMarker> {- match self {- Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {- hash_table- }- Self::Done => unreachable!("Done state does not hold a hash table"),- }+ fn has_spills(&self) -> bool {+ !self.spills.is_empty()}- fn into_producing_output(self) -> Self {- Self::ProducingOutput {- hash_table: self.into_hash_table(),- }+ /// Sorts and spills the aggregated groups. Memory reservation should be updated+ /// by the caller.+ ///+ /// Individual spill files are ordered by the `group by` keys.+ ///+ /// See [`SingleHashAggregateStream`] for spilling details.+ fn spill_table(+ &mut self,+ hash_table: &mut AggregateHashTable<SingleMarker>,+ ) -> Result<()> {+ let Some(batch) = hash_table.take_state_batch()? else {+ return Ok(());+ };++ let sorted_iter =+ IncrementalSortIterator::new(batch, self.spill_expr.clone(), self.batch_size);+ let spill_file = self+ .spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(+ sorted_iter,+ "SingleHashAggregateSpill",+ )?;++ let Some((file, max_record_batch_memory)) = spill_file else {+ return internal_err!("Single hash aggregation produced an empty spill");+ };++ self.spills.push(SortedSpillFile {+ file,+ max_record_batch_memory,+ });++ Ok(())}- fn into_done(self) -> Self {- Self::Done+ /// Merges every sorted run, and do the aggregate evaluation with+ /// [`OrderedFinalAggregateStream`]+ fn into_replay_stream(+ self,+ baseline_metrics: &BaselineMetrics,+ group_by_metrics: GroupByMetrics,+ reservation: MemoryReservation,+ ) -> Result<SendableRecordBatchStream> {+ let Self {+ final_agg,+ context,+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills,+ } = self;++ let spill_schema = Arc::clone(spill_manager.schema());+ let merged = StreamingMergeBuilder::new()+ .with_schema(spill_schema)+ .with_spill_manager(spill_manager)+ .with_sorted_spill_files(spills)+ .with_expressions(&spill_expr)+ .with_metrics(baseline_metrics.intermediate())+ .with_batch_size(batch_size)+ .with_reservation(reservation)+ .build()?;+ let replay = OrderedFinalAggregateStream::new_with_input_and_metrics(+ &final_agg,+ &context,+ partition,+ merged,+ &InputOrderMode::Sorted,+ baseline_metrics.clone(),+ group_by_metrics,+ None,+ )?;+ Ok(Box::pin(replay))}}@@ -139,24 +312,48 @@ impl SingleHashAggregateStream {) -> Result<Self> {debug_assert!(matches!(agg.mode,- super::AggregateMode::Single | super::AggregateMode::SinglePartitioned+ AggregateMode::Single | AggregateMode::SinglePartitioned));debug_assert_eq!(agg.input_order_mode, InputOrderMode::Linear);let schema = Arc::clone(&agg.schema);let input = agg.input.execute(partition, Arc::clone(context))?;+ let input_schema = input.schema();let batch_size = context.session_config().batch_size();let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);+ let spill_metrics = SpillMetrics::new(&agg.metrics, partition);+ let state_schema = Arc::new(create_schema(+ input_schema.as_ref(),+ &agg.group_by,+ &agg.aggr_expr,+ AggregateMode::Partial,+ )?);let hash_table = AggregateHashTable::<SingleMarker>::new(agg,partition,Arc::clone(&schema),+ Arc::clone(&state_schema),batch_size,)?;+ let can_spill = context.runtime_env().disk_manager.tmp_files_enabled();+ let spill_context = if can_spill {+ Some(Box::new(SingleSpillContext::new(+ agg,+ context,+ partition,+ batch_size,+ &state_schema,+ spill_metrics,+ )?))+ } else {+ None+ };+let reservation =MemoryConsumer::new(format!("SingleHashAggregateStream[{partition}]"))+ .with_can_spill(can_spill).register(context.memory_pool());Ok(Self {@@ -164,24 +361,48 @@ impl SingleHashAggregateStream {input,baseline_metrics,reservation,- state: Some(SingleHashAggregateState::ReadingInput { hash_table }),+ state: Some(SingleHashAggregateState::ReadingInput {+ hash_table,+ spill_context,+ }),})}- /// Moves the aggregate hash table's inner state to `Outputting`.- ///- /// The caller guarantees that input is fully consumed, so this function can- /// eagerly release the input stream.- fn start_output(- &mut self,- hash_table: &mut AggregateHashTable<SingleMarker>,- ) -> Result<()> {+ fn close_input(&mut self) {let input_schema = self.input.schema();self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));- hash_table.start_output()}- /// Handle ReadingInput state - aggregate input batches into the hash table.+ fn break_with_err(error: DataFusionError) -> SingleHashAggregateStateTransition {+ ControlFlow::Break((+ Poll::Ready(Some(Err(error))),+ SingleHashAggregateState::Error,+ ))+ }++ fn break_with_internal_err(message: &str) -> SingleHashAggregateStateTransition {+ Self::break_with_err(internal_datafusion_err!("{message}"))+ }++ /// Reserve memory for the current aggregate table.+ fn reservation_size_for_table(+ hash_table: &AggregateHashTable<SingleMarker>,+ spill_context: Option<&SingleSpillContext>,+ ) -> usize {+ let table_size = hash_table.memory_size();+ if spill_context.is_some() {+ // See `SingleHashAggregateStream` comments for how this is estimated.+ table_size.saturating_add(+ hash_table+ .building_group_count()+ .saturating_mul(size_of::<u32>()),+ )+ } else {+ table_size+ }+ }++ /// Consumes one raw input batch and updates the single-stage hash table.////// See comments at `poll_next()` for details.///@@ -189,100 +410,272 @@ impl SingleHashAggregateStream {fn handle_reading_input(&mut self,cx: &mut Context<'_>,- mut original_state: SingleHashAggregateState,+ original_state: SingleHashAggregateState,) -> SingleHashAggregateStateTransition {- debug_assert!(matches!(- &original_state,- SingleHashAggregateState::ReadingInput { .. }- ));- debug_assert!(original_state.hash_table().is_building());+ let SingleHashAggregateState::ReadingInput {+ mut hash_table,+ spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Single hash aggregate stream expected ReadingInput state",+ );+ };match self.input.poll_next_unpin(cx) {- Poll::Pending => ControlFlow::Break((Poll::Pending, original_state)),- // Get a new input batch, aggregate it in the hash table+ Poll::Pending => ControlFlow::Break((+ Poll::Pending,+ SingleHashAggregateState::ReadingInput {+ hash_table,+ spill_context,+ },+ )),Poll::Ready(Some(Ok(batch))) => {let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();let timer = elapsed_compute.timer();- let result = original_state.hash_table_mut().aggregate_batch(&batch);+ let result = hash_table.aggregate_batch(&batch);timer.done();if let Err(e) = result {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- original_state,- ));+ return Self::break_with_err(e);}- if let Err(e) = self- .reservation- .try_resize(original_state.hash_table().memory_size())- {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- original_state,- ));+ // Check memory reservation, and potentially spill.+ let timer = elapsed_compute.timer();+ let resize_result =+ self.reservation+ .try_resize(Self::reservation_size_for_table(+ &hash_table,+ spill_context.as_deref(),+ ));+ timer.done();+ match resize_result {+ Ok(()) => {}+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {+ let Some(spill_context) = spill_context else {+ return Self::break_with_err(e.context(+ "Single hash aggregate cannot spill because temporary files are not enabled in the DiskManager",+ ));+ };+ if hash_table.building_group_count() == 0 {+ return Self::break_with_internal_err(+ "Single hash aggregate ran out of memory with no aggregated groups",+ );+ }+ return ControlFlow::Continue(+ SingleHashAggregateState::Spilling {+ hash_table,+ spill_context,+ },+ );+ }+ Err(e) => {+ return Self::break_with_err(e);+ }}- ControlFlow::Continue(original_state)+ ControlFlow::Continue(SingleHashAggregateState::ReadingInput {+ hash_table,+ spill_context,+ })}- Poll::Ready(Some(Err(e))) => {- ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state))- }- // Input ends, move to output state+ Poll::Ready(Some(Err(e))) => Self::break_with_err(e),Poll::Ready(None) => {- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- let timer = elapsed_compute.timer();- let result = self.start_output(original_state.hash_table_mut());- timer.done();-- match result {- Ok(()) => {- ControlFlow::Continue(original_state.into_producing_output())+ self.close_input();+ match spill_context {+ Some(spill_context) if spill_context.has_spills() => {+ ControlFlow::Continue(+ SingleHashAggregateState::PreparingMergeInput {+ hash_table,+ spill_context,+ },+ )}- Err(e) => {- ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state))+ _ => {+ let elapsed_compute =+ self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let result = hash_table.start_output();+ timer.done();++ match result {+ Ok(()) => ControlFlow::Continue(+ SingleHashAggregateState::ProducingOutput { hash_table },+ ),+ Err(e) => Self::break_with_err(e),+ }}}}}}- /// Handle ProducingOutput state - emit final aggregate value batches.+ /// Sorts and spills one complete in-memory state run, then resumes input.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_spilling(+ &mut self,+ original_state: SingleHashAggregateState,+ ) -> SingleHashAggregateStateTransition {+ let SingleHashAggregateState::Spilling {+ mut hash_table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Single hash aggregate stream expected Spilling state",+ );+ };++ // Sanity check: it is impossible to OOM when the table is empty.+ if hash_table.building_group_count() == 0 {+ return Self::break_with_internal_err(+ "Single hash aggregation entered Spilling with an empty table",+ );+ }++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let mut result = spill_context.spill_table(&mut hash_table);++ // Spilling shrinks the aggregate table and releases its accumulated+ // memory. Update the reservation accordingly.+ if let Err(e) = self.reservation.try_resize(hash_table.memory_size()) {+ result =+ Err(e.context("Decreasing allocation after spilling should succeed"));+ }++ timer.done();++ match result {+ // Finished spilling the aggregate table, continue aggregating from input.+ Ok(()) => ControlFlow::Continue(SingleHashAggregateState::ReadingInput {+ hash_table,+ spill_context: Some(spill_context),+ }),+ Err(e) => Self::break_with_err(e),+ }+ }++ /// 1. Spills the last in-memory run.+ /// 2. Constructs a globally ordered input stream by applying a sort-preserving+ /// merge to all spills.+ /// 3. Constructs a replay stream: an ordered final aggregate stream over the+ /// fully ordered input constructed from the spills.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_preparing_merge_input(+ &mut self,+ original_state: SingleHashAggregateState,+ ) -> SingleHashAggregateStateTransition {+ let SingleHashAggregateState::PreparingMergeInput {+ mut hash_table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Single hash aggregate stream expected PreparingMergeInput state",+ );+ };++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let replay = match spill_context.spill_table(&mut hash_table) {+ Ok(()) => {+ let group_by_metrics = hash_table.group_by_metrics().clone();+ drop(hash_table);+ match self.reservation.try_resize(0) {+ Ok(()) => (*spill_context).into_replay_stream(+ &self.baseline_metrics,+ group_by_metrics,+ self.reservation.new_empty(),+ ),+ Err(e) => Err(e),+ }+ }+ Err(e) => Err(e),+ };+ timer.done();++ match replay {+ Ok(stream) => {+ ControlFlow::Continue(SingleHashAggregateState::MergingSpills { stream })+ }+ Err(e) => Self::break_with_err(e),+ }+ }++ /// Forwards output from the fully ordered stream that consumes the merged+ /// spill runs.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_merging_spills(+ &mut self,+ cx: &mut Context<'_>,+ original_state: SingleHashAggregateState,+ ) -> SingleHashAggregateStateTransition {+ let SingleHashAggregateState::MergingSpills { mut stream } = original_state+ else {+ return Self::break_with_internal_err(+ "Single hash aggregate stream expected MergingSpills state",+ );+ };++ match stream.poll_next_unpin(cx) {+ Poll::Pending => ControlFlow::Break((+ Poll::Pending,+ SingleHashAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Ok(batch))) => ControlFlow::Break((+ Poll::Ready(Some(Ok(batch))),+ SingleHashAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Err(e))) => Self::break_with_err(e),+ Poll::Ready(None) => ControlFlow::Continue(SingleHashAggregateState::Done),+ }+ }++ /// Emits one batch after input is exhausted.////// See comments at `poll_next()` for details.////// Returns the next operator state with control flow decision.fn handle_producing_output(&mut self,- mut original_state: SingleHashAggregateState,+ original_state: SingleHashAggregateState,) -> SingleHashAggregateStateTransition {- debug_assert!(matches!(- &original_state,- SingleHashAggregateState::ProducingOutput { .. }- ));- debug_assert!(!original_state.hash_table().is_building());+ let SingleHashAggregateState::ProducingOutput { mut hash_table } = original_state+ else {+ return Self::break_with_internal_err(+ "Single hash aggregate stream expected ProducingOutput state",+ );+ };let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();let timer = elapsed_compute.timer();- let result = original_state.hash_table_mut().next_output_batch();+ let result = hash_table.next_output_batch();timer.done();match result {Ok(Some(batch)) => {- if let Err(e) = self- .reservation- .try_resize(original_state.hash_table().memory_size())- {- return ControlFlow::Break((- Poll::Ready(Some(Err(e))),- original_state,- ));- }- debug_assert!(batch.num_rows() > 0);- let next_state = if original_state.hash_table().is_done() {- original_state.into_done()+ let next_state = if hash_table.is_done() {+ drop(hash_table);+ if let Err(e) = self.reservation.try_resize(0) {+ return Self::break_with_err(e);+ }+ SingleHashAggregateState::Done} else {- original_state+ if let Err(e) = self.reservation.try_resize(hash_table.memory_size())+ {+ return Self::break_with_err(e);+ }+ SingleHashAggregateState::ProducingOutput { hash_table }};ControlFlow::Break((@@ -290,11 +683,15 @@ impl SingleHashAggregateStream {next_state,))}+ Err(e) => Self::break_with_err(e),Ok(None) => {- let _ = self.reservation.try_resize(0);- ControlFlow::Continue(original_state.into_done())+ drop(hash_table);+ let next_state = SingleHashAggregateState::Done;+ if let Err(e) = self.reservation.try_resize(0) {+ return Self::break_with_err(e);+ }+ ControlFlow::Continue(next_state)}- Err(e) => ControlFlow::Break((Poll::Ready(Some(Err(e))), original_state)),}}}@@ -316,21 +713,48 @@ impl Stream for SingleHashAggregateStream {////// ReadingInput/// -> ReadingInput- /// Aggregate one raw input batch, update the inner aggregate hash- /// table, and continue with the next input batch.- ///+ /// Aggregate one raw input batch. If it fits in memory, continue with+ /// the next input batch.+ /// -> Spilling+ /// The table cannot reserve enough memory. Move all current states into+ /// one fully group-key-sorted spill run./// -> ProducingOutput- /// Input was exhausted. Move to the next state to start outputting- /// final aggregate values.+ /// Input was exhausted without spilling. Start outputting final values.+ /// -> PreparingMergeInput+ /// Input was exhausted after spilling. Spill the last in-memory run and+ /// construct the ordered input used to merge all spill files.+ ///+ /// Spilling+ /// -> ReadingInput+ /// One sorted run was written; resume reading the original input.+ ///+ /// PreparingMergeInput+ /// Spill the final in-memory run and build the input ordered replay stream.+ /// -> MergingSpills+ /// The final run was spilled and the ordered replay stream was built.+ ///+ /// MergingSpills+ /// Aggregate the merged spill runs and emit final results.+ /// -> MergingSpills+ /// Forward one result batch from the fully ordered replay stream that+ /// consumes the sort-preserving merge.+ /// -> Done+ /// The merged spill input was fully aggregated.////// ProducingOutput/// -> ProducingOutput/// One final output batch was yielded; repeat to continue producing/// output incrementally.- ////// -> Done/// All final output was emitted.///+ /// Any active state+ /// -> Error+ /// An error drops state-owned resources before it is returned.+ ///+ /// Error+ /// -> (end)+ ////// Done/// -> (end)/// ```@@ -348,9 +772,24 @@ impl Stream for SingleHashAggregateStream {state @ SingleHashAggregateState::ReadingInput { .. } => {self.handle_reading_input(cx, state)}+ state @ SingleHashAggregateState::Spilling { .. } => {+ self.handle_spilling(state)+ }+ state @ SingleHashAggregateState::PreparingMergeInput { .. } => {+ self.handle_preparing_merge_input(state)+ }+ state @ SingleHashAggregateState::MergingSpills { .. } => {+ self.handle_merging_spills(cx, state)+ }state @ SingleHashAggregateState::ProducingOutput { .. } => {self.handle_producing_output(state)}+ state @ SingleHashAggregateState::Error => {+ self.close_input();+ self.reservation.free();+ self.state = Some(state);+ return Poll::Ready(None);+ }state @ SingleHashAggregateState::Done => {let _ = self.reservation.try_resize(0);self.state = Some(state);@@ -363,6 +802,16 @@ impl Stream for SingleHashAggregateStream {self.state = Some(next_state);continue;}+ ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {+ debug_assert!(matches!(next_state, SingleHashAggregateState::Error));++ // The handler has already discarded its state-owned resources.+ // Release the remaining stream-owned resources before returning.+ self.close_input();+ self.reservation.free();+ self.state = Some(SingleHashAggregateState::Error);+ return Poll::Ready(Some(Err(e)));+ }ControlFlow::Break((poll, next_state)) => {self.state = Some(next_state);return poll;diff --git a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt b/datafusion/sqllogictest/test_files/aggregate_memory_spill.sltindex 7615209255..cce3a3e903 100644--- a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt+++ b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt@@ -33,6 +33,9 @@statement okSET datafusion.execution.target_partitions = 1+statement ok+SET datafusion.execution.batch_size = 128+statement okSET datafusion.runtime.memory_limit = '1M'@@ -47,8 +50,7 @@ FROM (----100000 5000050000-# Prove the inner aggregate actually spills (else these tests would silently stop covering the spill path).-# Only `spill_count` is pinned; the other metrics vary per run.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), sum(total)@@ -58,13 +60,9 @@ FROM (GROUP BY (v * 7) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(total)@1 as sum(total)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(total)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[sum(t.v)@1 as total], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=9,<slt:ignore>]-05)--------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># --- Case B: multi-column GROUP BY (is_single() = false) ---# Both keys are bijections of v, so each (a, b) pair is unique: 100000 groups.@@ -78,7 +76,7 @@ FROM (----100000 5000050000-# Assert this case spills too.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), sum(total)@@ -88,13 +86,9 @@ FROM (GROUP BY (v * 7) % 100000, (v * 13) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(total)@1 as sum(total)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(total)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[sum(t.v)@2 as total], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000), v@0 * 13 % 100000 as t.v * Int64(13) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=11,<slt:ignore>]-05)--------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000), v@0 * 13 % 100000 as t.v * Int64(13) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># --- Case C: DISTINCT aggregate under memory limit ---# One distinct value per group, so each count(DISTINCT v) = 1.@@ -108,7 +102,7 @@ FROM (----100000 100000-# Assert this case spills too.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), sum(d)@@ -118,14 +112,9 @@ FROM (GROUP BY (v * 7) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(d)@1 as sum(d)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(d)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[count(alias1)@1 as d], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[group_alias_0@0 as group_alias_0], aggr=[count(alias1)], metrics=[<slt:ignore>spill_count=18,<slt:ignore>]-05)--------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as group_alias_0, v@0 as alias1], aggr=[], ordering_mode=Sorted, metrics=[<slt:ignore>]-06)----------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[group_alias_0@0 as group_alias_0], aggr=[count(alias1)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># --- Case D: multiple aggregates (sum/min/max) under memory limit ---# Each group holds a single v, so min(v) = max(v) = v within the group.@@ -139,7 +128,7 @@ FROM (----100000 5000050000 1 100000-# Assert this case spills too.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), sum(s), min(mn), max(mx)@@ -149,13 +138,9 @@ FROM (GROUP BY (v * 7) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(s)@1 as sum(s), min(mn)@2 as min(mn), max(mx)@3 as max(mx)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(s), min(mn), max(mx)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[sum(t.v)@1 as s, min(t.v)@2 as mn, max(t.v)@3 as mx], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v), min(t.v), max(t.v)], metrics=[<slt:ignore>spill_count=27,<slt:ignore>]-05)--------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v), min(t.v), max(t.v)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># --- Case E: avg() aggregate (Float64 output) under memory limit ---# Each group holds a single v, so avg(v) = v within the group.@@ -169,7 +154,7 @@ FROM (----100000 1 100000-# Assert this case spills too.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), min(a), max(a)@@ -179,13 +164,9 @@ FROM (GROUP BY (v * 7) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), min(a)@1 as min(a), max(a)@2 as max(a)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), min(a), max(a)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[avg(t.v)@1 as a], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[avg(t.v)], metrics=[<slt:ignore>spill_count=11,<slt:ignore>]-05)--------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[avg(t.v)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># --- Case F: array_agg() aggregate (growable state) under memory limit ---# Each group holds a single v, so array_length(array_agg(v)) = 1.@@ -199,7 +180,7 @@ FROM (----100000 100000-# Assert this case spills too.+# Assert spill happened, the `spill_count` metric must be > 0query TTEXPLAIN ANALYZESELECT count(*), sum(l)@@ -209,18 +190,17 @@ FROM (GROUP BY (v * 7) % 100000)-----Plan with Metrics-01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(l)@1 as sum(l)], metrics=[<slt:ignore>]-02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(l)], metrics=[<slt:ignore>]-03)----ProjectionExec: expr=[array_length(array_agg(t.v)@1) as l], metrics=[<slt:ignore>]-04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[array_agg(t.v)], metrics=[<slt:ignore>spill_count=10,<slt:ignore>]-05)--------ProjectionExec: expr=[value@0 as v], metrics=[<slt:ignore>]-06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[<slt:ignore>]+<slt:ignore>+04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[array_agg(t.v)], metrics=[<slt:ignore>spill_count=7,<slt:ignore>]+<slt:ignore># Restore settings to slt runner defaultsstatement okRESET datafusion.runtime.memory_limit+statement ok+RESET datafusion.execution.batch_size+statement okSET datafusion.execution.target_partitions = 4