Apache DataFusion SQL Query Engine
refactor(hash-aggr): migrate ordered single aggregation (#24259)
Yongting You186f96fba6b0
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rsindex 2293e7b1b8..dac0d4b7c5 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs@@ -38,9 +38,12 @@ use crate::aggregates::{evaluate_group_by,};-use super::common::{AggregateAccumulator, EvaluatedAggregateBatch};+use super::common::{+ AggregateAccumulator, AggregateBatchFn, EvaluatedAggregateBatch,+ MaterializeAccumulatorFn,+};-/// Aggregate table shared by the ordered partial and final paths.+/// Aggregate table shared by the ordered single, partial and final paths.////// # Ordering optimization///@@ -48,10 +51,11 @@ use super::common::{AggregateAccumulator, EvaluatedAggregateBatch};/// are proven complete. Completed groups can be emitted before the input stream/// ends, which keeps memory bounded by the active ordered key range.///-/// # Partial and final variant difference+/// # Single, partial and final variant difference////// The partial and final aggregate tables implement the two stages of grouped-/// aggregation. See+/// aggregation, while the single aggregate table implements both stages in one+/// table. See/// [`OrderedPartialAggregateStream`](crate::aggregates::ordered_partial_stream::OrderedPartialAggregateStream)/// for the high-level plan shape.///@@ -67,6 +71,11 @@ use super::common::{AggregateAccumulator, EvaluatedAggregateBatch};/// - Table stores: `k, sum(v), count(v)`/// - Output schema: `k, avg(v)`///+/// Single table ([`AggregateMode::Single`], with optional filter from query):+/// - Input rows: `k, v`+/// - Table stores: `k, sum(v), count(v)`+/// - Output schema: `k, avg(v)`+////// # Marker Type////// `OrderedAggrMode` selects the aggregate semantics. For example,@@ -75,7 +84,7 @@ use super::common::{AggregateAccumulator, EvaluatedAggregateBatch};/// `OrderedAggregateTable::<FinalMarker>::new_with_input_order(...)`/// consumes partial states and emits final values.///-/// Shared methods live on `impl<T>`; partial/final behavior lives on+/// Shared methods live on `impl<T>`; single/partial/final behavior lives on/// marker-specific impls.pub(in crate::aggregates) struct OrderedAggregateTable<OrderedAggrMode> {/// Output schema: group columns followed by aggregate state or final values.@@ -129,7 +138,7 @@ pub(super) struct OrderedAggregateTableBuffer {impl<AggrMode> OrderedAggregateTable<AggrMode> {#[expect(clippy::too_many_arguments,- reason = "keeps ordered partial and final table construction explicit"+ reason = "keeps ordered single, partial and final table construction explicit")]pub(super) fn new_for_mode(agg: &AggregateExec,@@ -259,8 +268,8 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {////// Unlike normal ordered emission, this operation is allowed to take the/// active (incomplete) groups. Partial aggregation can pass those states to- /// its final stage, while final aggregation sorts and spills them before- /// replay.+ /// its final stage, while single and final aggregation sort and spill them+ /// before replay.pub(in crate::aggregates) fn take_state_batch(&mut self,) -> Result<Option<RecordBatch>> {@@ -304,18 +313,18 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {EmitTo::All => (EmitTo::First(self.batch_size), false),}}- /// Aggregates one evaluated input batch.- ///- /// This common utility is used by ordered partial and ordered final aggregation.- ///- /// # Argument: `is_final`++ /// Aggregates one evaluated input batch after selecting the mode-specific+ /// accumulator operation.///- /// - `true`: merge partial aggregate states for final aggregation.- /// - `false`: update aggregate states from raw input for partial aggregation.+ /// Each aggregation mode chooses a different `aggregate_fn` according to its+ /// semantics. For example, partial aggregation takes raw inputs and updates+ /// stored partial states, so it uses+ /// [`datafusion_expr::GroupsAccumulator::update_batch`].pub(super) fn aggregate_evaluated_batch(&mut self,evaluated_batch: &EvaluatedAggregateBatch,- is_final: bool,+ aggregate_fn: AggregateBatchFn,) -> Result<()> {for group_values in &evaluated_batch.grouping_set_args {let starting_num_groups = self.buffer.group_values.len();@@ -338,19 +347,7 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {.iter_mut().zip(evaluated_batch.accumulator_args.iter()){- if is_final {- acc.merge_batch(- values,- &self.buffer.group_indices,- total_num_groups,- )?;- } else {- acc.update_batch(- values,- &self.buffer.group_indices,- total_num_groups,- )?;- }+ aggregate_fn(acc, values, &self.buffer.group_indices, total_num_groups)?;}drop(timer);}@@ -361,15 +358,13 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {/// Emits groups allowed by `GroupOrdering`, leaving only the current/// unfinished ordered-key range buffered.///- /// This common utility is used by ordered partial and ordered final aggregation.- ///- /// # Argument: `is_final`- ///- /// - `true`: output final aggregate values.- /// - `false`: output partial accumulator states.- pub(super) fn next_output_batch_for_mode(+ /// Each aggregation mode chooses a different `materialize_accumulator_fn`+ /// according to its semantics. For example, partial aggregation emits+ /// partial states to feed the final stage, so it uses+ /// [`datafusion_expr::GroupsAccumulator::state`].+ pub(super) fn next_output_batch_inner(&mut self,- is_final: bool,+ materialize_accumulator_fn: MaterializeAccumulatorFn,) -> Result<Option<RecordBatch>> {if self.buffer.group_values.is_empty() {return Ok(None);@@ -394,11 +389,7 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {}for acc in &mut self.buffer.accumulators {- if is_final {- output.push(acc.evaluate(emit_to)?);- } else {- output.extend(acc.state(emit_to)?);- }+ output.extend(materialize_accumulator_fn(acc, emit_to)?);}drop(timer);diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rsindex 2c7ec01654..fbf3ccc738 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs@@ -20,6 +20,7 @@ mod common_ordered;mod final_table;mod ordered_final_table;mod ordered_partial_table;+mod ordered_single_table;mod partial_reduce_table;mod partial_table;mod single_table;diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rsindex fd064ebffe..1b7d419dd5 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs@@ -30,6 +30,7 @@ use crate::aggregates::aggregate_hash_table::FinalMarker;use crate::aggregates::group_values::GroupByMetrics;use crate::aggregates::{AggregateExec, AggregateMode};+use super::common::HashAggregateAccumulator;use super::common_ordered::OrderedAggregateTable;/// Implementation specific to final aggregation, where the table stores partial@@ -73,13 +74,16 @@ impl OrderedAggregateTable<FinalMarker> {// `PhysicalGroupBy::as_final()` removes grouping sets while planning// final aggregation, so final ordered aggregation sees one grouping.debug_assert_eq!(evaluated_batch.grouping_set_args.len(), 1);- self.aggregate_evaluated_batch(&evaluated_batch, true)+ self.aggregate_evaluated_batch(+ &evaluated_batch,+ HashAggregateAccumulator::merge_batch,+ )}/// See comments in `ordered_partial_stream::next_output_batch`pub(in crate::aggregates) fn next_output_batch(&mut self,) -> Result<Option<RecordBatch>> {- self.next_output_batch_for_mode(true)+ self.next_output_batch_inner(HashAggregateAccumulator::evaluate_to_columns)}}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rsindex a04e4dda8f..8545289f99 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs@@ -40,6 +40,7 @@ use crate::aggregates::{group_values::GroupByMetrics,};+use super::common::HashAggregateAccumulator;use super::common_ordered::OrderedAggregateTable;/// Implementation specific to partial aggregation, where the table stores@@ -81,7 +82,10 @@ impl OrderedAggregateTable<PartialMarker> {batch: &RecordBatch,) -> Result<()> {let evaluated_batch = self.evaluate_batch(batch)?;- self.aggregate_evaluated_batch(&evaluated_batch, false)+ self.aggregate_evaluated_batch(+ &evaluated_batch,+ HashAggregateAccumulator::update_batch,+ )}/// Emits the next batch of partial state rows for groups proven complete by@@ -101,6 +105,6 @@ impl OrderedAggregateTable<PartialMarker> {pub(in crate::aggregates) fn next_output_batch(&mut self,) -> Result<Option<RecordBatch>> {- self.next_output_batch_for_mode(false)+ self.next_output_batch_inner(HashAggregateAccumulator::state)}}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rsnew file mode 100644index 0000000000..8ba50e2a59--- /dev/null+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs@@ -0,0 +1,90 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Aggregate table for single aggregation when raw input is ordered.+//!+//! See comments in [`super::ordered_partial_table`] for details.++use arrow::datatypes::SchemaRef;+use arrow::record_batch::RecordBatch;+use datafusion_common::Result;++use crate::aggregates::aggregate_hash_table::SingleMarker;+use crate::aggregates::group_values::GroupByMetrics;+use crate::aggregates::{AggregateExec, AggregateMode};++use super::common::HashAggregateAccumulator;+use super::common_ordered::OrderedAggregateTable;++/// Implementation specific to single aggregation, where the table stores final+/// aggregate values and the input rows are raw rows.+///+/// Example: `AVG(x) GROUP BY k`+///+/// - Aggregate table stores: `k, avg(x)`+/// - Input rows: `k, x`+///+/// See comments at [`OrderedAggregateTable`] for details.+impl OrderedAggregateTable<SingleMarker> {+ pub(in crate::aggregates) fn new(+ agg: &AggregateExec,+ partition: usize,+ output_schema: SchemaRef,+ state_schema: SchemaRef,+ batch_size: usize,+ ) -> Result<Self> {+ debug_assert!(matches!(+ agg.mode,+ AggregateMode::Single | AggregateMode::SinglePartitioned+ ));++ let input_schema = agg.input().schema();+ let group_by_metrics = GroupByMetrics::new(&agg.metrics, partition);+ Self::new_for_mode(+ agg,+ &input_schema,+ output_schema,+ state_schema,+ batch_size,+ &agg.input_order_mode,+ &agg.mode,+ agg.filter_expr.iter().cloned().collect(),+ group_by_metrics,+ )+ }++ /// Aggregates one raw input batch and updates ordering information for any+ /// newly observed groups.+ pub(in crate::aggregates) fn aggregate_batch(+ &mut self,+ batch: &RecordBatch,+ ) -> Result<()> {+ let evaluated_batch = self.evaluate_batch(batch)?;+ self.aggregate_evaluated_batch(+ &evaluated_batch,+ HashAggregateAccumulator::update_batch,+ )+ }++ /// Emits the next batch of final aggregate values for groups proven complete+ /// by the input ordering.+ pub(in crate::aggregates) fn next_output_batch(+ &mut self,+ ) -> Result<Option<RecordBatch>> {+ self.next_output_batch_inner(HashAggregateAccumulator::evaluate_to_columns)+ }+}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex d0aadc2847..f9dd90f6f9 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -59,8 +59,8 @@//! AggregateExec (partial, ordered)//! ```//!-//! See [`OrderedPartialAggregateStream`] and [`OrderedFinalAggregateStream`] for-//! details.+//! See [`OrderedPartialAggregateStream`], [`OrderedFinalAggregateStream`], and+//! [`OrderedSingleAggregateStream`] for details.//!//! Related configuration://!@@ -81,7 +81,8 @@//! input//! ```//!-//! See [`SingleHashAggregateStream`] for details.+//! See [`SingleHashAggregateStream`] and [`OrderedSingleAggregateStream`] for+//! details.//!//! Related configuration://!@@ -153,6 +154,7 @@ use crate::aggregates::{hash_stream::{FinalHashAggregateStream, PartialHashAggregateStream},ordered_final_stream::OrderedFinalAggregateStream,ordered_partial_stream::OrderedPartialAggregateStream,+ ordered_single_stream::OrderedSingleAggregateStream,partial_reduce_stream::PartialReduceHashAggregateStream,single_stream::SingleHashAggregateStream,};@@ -213,6 +215,7 @@ mod hash_stream;pub mod order;mod ordered_final_stream;mod ordered_partial_stream;+mod ordered_single_stream;mod partial_reduce_stream;mod single_stream;mod skip_partial;@@ -676,12 +679,15 @@ enum StreamType {OrderedPartialAggregate(OrderedPartialAggregateStream),/// Final stage of aggregation for ordered input.OrderedFinalAggregate(OrderedFinalAggregateStream),+ /// Single stage of aggregation for ordered input.+ OrderedSingleAggregate(OrderedSingleAggregateStream),/// Hash aggregation reused for multiple stages////// Note this is being incrementally migrated to dedicated streams like/// [`StreamType::PartialHash`], [`StreamType::FinalHash`],- /// [`StreamType::OrderedPartialAggregate`], and- /// [`StreamType::OrderedFinalAggregate`]+ /// [`StreamType::OrderedPartialAggregate`],+ /// [`StreamType::OrderedFinalAggregate`], and+ /// [`StreamType::OrderedSingleAggregate`]////// See issue for details: <https://github.com/apache/datafusion/issues/22710>GroupedHash(GroupedHashAggregateStream),@@ -703,6 +709,7 @@ impl From<StreamType> for SendableRecordBatchStream {StreamType::SingleHash(stream) => Box::pin(stream),StreamType::OrderedPartialAggregate(stream) => stream.into_stream(),StreamType::OrderedFinalAggregate(stream) => Box::pin(stream),+ StreamType::OrderedSingleAggregate(stream) => Box::pin(stream),StreamType::GroupedHash(stream) => Box::pin(stream),StreamType::GroupedPriorityQueue(stream) => Box::pin(stream),}@@ -1216,6 +1223,12 @@ impl AggregateExec {)?));}+ if self.should_use_ordered_single_aggregate_stream(context) {+ return Ok(StreamType::OrderedSingleAggregate(+ OrderedSingleAggregateStream::new(self, context, partition)?,+ ));+ }+if self.should_use_single_hash_stream(context) {return Ok(StreamType::SingleHash(SingleHashAggregateStream::new(self, context, partition,@@ -1281,6 +1294,16 @@ impl AggregateExec {&& self.group_by.is_single()}+ fn should_use_ordered_single_aggregate_stream(&self, _context: &TaskContext) -> bool {+ matches!(+ self.mode,+ AggregateMode::Single | AggregateMode::SinglePartitioned+ ) && self.limit_options.is_none()+ && self.input_order_mode != InputOrderMode::Linear+ && !self.group_by.is_true_no_grouping()+ && self.group_by.is_single()+ }+fn should_use_ordered_final_aggregate_stream(&self, _context: &TaskContext) -> bool {matches!(self.mode,@@ -4264,6 +4287,83 @@ mod tests {Ok(())}+ /// Ensures `OrderedSingleAggregateStream` is used for ordered raw input.+ #[tokio::test]+ async fn ordered_single_aggregate_planning() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("sort_col", DataType::Int32, false),+ Field::new("group_col", DataType::Int32, false),+ Field::new("value_col", DataType::Int64, false),+ ]));+ let input_batches = vec![+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 1, 1])),+ Arc::new(Int32Array::from(vec![10, 11, 10])),+ Arc::new(Int64Array::from(vec![1, 2, 3])),+ ],+ )?,+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![2, 2])),+ Arc::new(Int32Array::from(vec![20, 21])),+ Arc::new(Int64Array::from(vec![4, 5])),+ ],+ )?,+ ];+ let ordering = LexOrdering::new([PhysicalSortExpr::new_default(Arc::new(+ Column::new("sort_col", 0),+ ))])+ .unwrap();+ let input = TestMemoryExec::try_new(&[input_batches], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![ordering])?;+ let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(input)));+ let aggregate = AggregateExec::try_new(+ AggregateMode::Single,+ PhysicalGroupBy::new_single(vec![+ (col("sort_col", &schema)?, "sort_col".to_string()),+ (col("group_col", &schema)?, "group_col".to_string()),+ ]),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("value_col", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("SUM(value_col)")+ .build()?,+ )],+ vec![None],+ input,+ Arc::clone(&schema),+ )?;+ assert!(matches!(+ aggregate.input_order_mode(),+ InputOrderMode::PartiallySorted(_)+ ));++ let task_ctx = new_migrated_hash_ctx(2);+ let stream = aggregate.execute_typed(0, &task_ctx)?;+ assert!(matches!(stream, StreamType::OrderedSingleAggregate(_)));+ let stream: SendableRecordBatchStream = stream.into();+ let output = collect(stream).await?;+ assert_snapshot!(batches_to_sort_string(&output), @r"++----------+-----------+----------------++| sort_col | group_col | SUM(value_col) |++----------+-----------+----------------++| 1 | 10 | 4 |+| 1 | 11 | 2 |+| 2 | 20 | 4 |+| 2 | 21 | 5 |++----------+-----------+----------------++");++ let finite_memory_task_ctx = new_finite_memory_migrated_hash_ctx(2, 1024 * 1024)?;+ let stream = aggregate.execute_typed(0, &finite_memory_task_ctx)?;+ assert!(matches!(stream, StreamType::OrderedSingleAggregate(_)));++ Ok(())+ }+fn partial_reduce_test_aggregate() -> Result<AggregateExec> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::UInt32, false),diff --git a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rsnew file mode 100644index 0000000000..2025bdce30--- /dev/null+++ b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs@@ -0,0 +1,886 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Single-stage aggregate stream for ordered raw input.++use std::ops::ControlFlow;+use std::sync::Arc;+use std::task::{Context, Poll};++use arrow::datatypes::SchemaRef;+use arrow::record_batch::RecordBatch;+use datafusion_common::{DataFusionError, Result, internal_datafusion_err, internal_err};+use datafusion_execution::TaskContext;+use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr_common::sort_expr::LexOrdering;+use futures::stream::{Stream, StreamExt};++use super::aggregate_hash_table::{OrderedAggregateTable, SingleMarker};+use super::group_values::GroupByMetrics;+use super::ordered_final_stream::OrderedFinalAggregateStream;+use super::{AggregateExec, create_schema};+use crate::aggregates::AggregateMode;+use crate::metrics::{BaselineMetrics, RecordOutput, SpillMetrics};+use crate::sorts::IncrementalSortIterator;+use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};+use crate::spill::spill_manager::SpillManager;+use crate::stream::EmptyRecordBatchStream;+use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};++/// Single aggregate stream for `InputOrderMode::Sorted` and+/// `InputOrderMode::PartiallySorted`.+///+/// # Example+///+/// SELECT k, AVG(v) FROM t GROUP BY k;+///+/// If the input is ordered by `k`, and there are existing key partitioning on group+/// by keys, the single mode aggregation with ordering optimization can be used:+///+/// ## Plan+/// AggregateExec(stage=single, ordered)+/// -- DataSourceExec(t)+///+/// ## Single Stage Behavior+/// Input: raw rows+/// Output: final results for all groups (for example, `AVG(x)`)+///+/// # Order-based Optimization+///+/// For the aggregation work, the hash aggregation implementation is reused.+///+/// After each input batch, check whether any groups can be emitted eagerly to+/// improve memory efficiency. For example, if the last group key seen is+/// `k = 100`, it is safe to emit all groups with keys less than 100 because the+/// input is ordered.+///+/// # Memory Pressure and Spilling+///+/// ## Fully ordered case+///+/// If the input is ordered by every group key, for example:+///+/// - Input order: `a, b`+/// - `GROUP BY`: `a, b`+///+/// Completed groups can be emitted as soon as the next group is observed. Thus,+/// only the current group remains active after completed groups are emitted, and+/// memory usage does not grow with the total number of groups.+///+/// If a memory reservation nevertheless fails, the stream returns the error+/// directly, indicating an unexpected behavior.+///+/// ## Partially ordered case+///+/// If the input is ordered by only a subset of the group keys, for example:+///+/// - Input order: `a`+/// - `GROUP BY`: `a, b`+///+/// If one `a` value contains many distinct `b` values, the table may accumulate+/// enough groups to exceed the memory limit.+///+/// On reservation failure, the stream sorts the current intermediate states by+/// the complete group key and spills them as one run. After the input ends, it+/// spills any remaining states, performs a sort-preserving merge of all runs,+/// and feeds the merged input into a fully ordered final aggregate stream.+pub(crate) struct OrderedSingleAggregateStream {+ schema: SchemaRef,+ input: SendableRecordBatchStream,+ reservation: MemoryReservation,+ baseline_metrics: BaselineMetrics,+ state: Option<OrderedSingleAggregateState>,+}++/// Spill configuration and accumulated runs for partially ordered single+/// aggregation.+///+/// Each spill event drains all currently buffered groups, sorts their intermediate+/// states by the full group key, and writes them to one spill file. All files are+/// merged and replayed after the original input ends.+struct OrderedSingleSpillContext {+ /// Aggregate configuration used to construct the final replay stream.+ final_agg: AggregateExec,+ /// Task context+ context: Arc<TaskContext>,+ /// Original partition index+ partition: usize,+ /// Target batch size from configuration+ batch_size: usize,+ /// Full group-key ordering, such ordering with be kept in: a) individual spill+ /// files, b) order after final merging and streaming aggregate+ spill_expr: LexOrdering,+ /// Spill I/O and metrics manager.+ spill_manager: SpillManager,+ /// Fully sorted spill runs waiting to be merged.+ spills: Vec<SortedSpillFile>,+}++/// See comments at `poll_next()` for details.+enum OrderedSingleAggregateState {+ ReadingInput {+ table: OrderedAggregateTable<SingleMarker>,+ /// None if either+ /// - Disk Manager doesn't enable temporary file creation+ /// - The group keys are fully ordered, it's expected to use bounded memory+ spill_context: Option<Box<OrderedSingleSpillContext>>,+ },+ Spilling {+ table: OrderedAggregateTable<SingleMarker>,+ spill_context: Box<OrderedSingleSpillContext>,+ },+ ProducingOutput {+ table: OrderedAggregateTable<SingleMarker>,+ },+ PreparingMergeInput {+ table: OrderedAggregateTable<SingleMarker>,+ spill_context: Box<OrderedSingleSpillContext>,+ },+ MergingSpills {+ stream: SendableRecordBatchStream,+ },+ Done,+ /// Sentinel state to use when returning error from any other states, because:+ /// - It explicitly releases state-owned resources immediately+ /// - More defensive against accidentally resuming execution after error+ Error,+}++type OrderedSingleAggregatePoll = Poll<Option<Result<RecordBatch>>>;+type OrderedSingleAggregateStateTransition = ControlFlow<+ (OrderedSingleAggregatePoll, OrderedSingleAggregateState),+ OrderedSingleAggregateState,+>;++impl OrderedSingleSpillContext {+ fn new(+ agg: &AggregateExec,+ context: &Arc<TaskContext>,+ partition: usize,+ batch_size: usize,+ input_order_mode: &InputOrderMode,+ spill_schema: &SchemaRef,+ spill_metrics: SpillMetrics,+ ) -> Result<Self> {+ let group_schema = agg.group_by.group_schema(&agg.input().schema())?;+ let output_ordering = agg.cache.output_ordering();+ let InputOrderMode::PartiallySorted(order_indices) = input_order_mode else {+ return internal_err!(+ "Ordered single spill requires partially ordered input"+ );+ };+ let spill_indices = order_indices.iter().copied().chain(+ (0..group_schema.fields().len()).filter(|idx| !order_indices.contains(idx)),+ );+ let spill_sort_exprs = spill_indices.map(|idx| {+ let field = group_schema.field(idx);+ let output_expr = Column::new(field.name(), idx);+ let sort_options = output_ordering+ .and_then(|ordering| ordering.get_sort_options(&output_expr))+ .unwrap_or_default();+ PhysicalSortExpr::new(Arc::new(output_expr), sort_options)+ });+ let Some(spill_expr) = LexOrdering::new(spill_sort_exprs) else {+ return internal_err!("Ordered single spill expression is empty");+ };++ let spill_manager = SpillManager::new(+ context.runtime_env(),+ spill_metrics,+ Arc::clone(spill_schema),+ )+ .with_compression_type(context.session_config().spill_compression());++ // Spilled rows contain group keys and intermediate states. Replay must+ // merge those states and evaluate the final aggregate values.+ let mut final_agg = agg.clone();+ final_agg.mode = match agg.mode {+ AggregateMode::Single => AggregateMode::Final,+ AggregateMode::SinglePartitioned => AggregateMode::FinalPartitioned,+ mode => {+ return internal_err!(+ "Ordered single aggregate spill cannot replay aggregate mode {mode:?}"+ );+ }+ };+ final_agg.group_by = Arc::new(agg.group_by.as_final());+ final_agg.input_order_mode = InputOrderMode::Sorted;++ Ok(Self {+ final_agg,+ context: Arc::clone(context),+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills: vec![],+ })+ }++ fn has_spills(&self) -> bool {+ !self.spills.is_empty()+ }++ /// Sorts and spills the aggregated groups. Memory reservation should be updated+ /// by the caller.+ ///+ /// Individual spill files are ordered by the `group by` keys.+ ///+ /// See [`OrderedSingleAggregateStream`] for spilling details.+ fn spill_table(+ &mut self,+ table: &mut OrderedAggregateTable<SingleMarker>,+ ) -> Result<()> {+ let Some(batch) = table.take_state_batch()? else {+ return Ok(());+ };++ let sorted_iter =+ IncrementalSortIterator::new(batch, self.spill_expr.clone(), self.batch_size);+ let spill_file = self+ .spill_manager+ .spill_record_batch_iter_and_return_max_batch_memory(+ sorted_iter,+ "OrderedSingleAggregateSpill",+ )?;++ let Some((file, max_record_batch_memory)) = spill_file else {+ return internal_err!("Ordered single aggregation produced an empty spill");+ };++ self.spills.push(SortedSpillFile {+ file,+ max_record_batch_memory,+ });++ Ok(())+ }++ /// Merges every sorted run and finalizes it through the fully ordered path.+ fn into_replay_stream(+ self,+ baseline_metrics: &BaselineMetrics,+ group_by_metrics: GroupByMetrics,+ reservation: MemoryReservation,+ ) -> Result<SendableRecordBatchStream> {+ let Self {+ final_agg,+ context,+ partition,+ batch_size,+ spill_expr,+ spill_manager,+ spills,+ } = self;++ let spill_schema = Arc::clone(spill_manager.schema());+ // The merge and replay table are two components of the same aggregate+ // operator. Keep them under one consumer registration so a fair memory+ // pool does not divide this operator's quota between its own phases.+ let merge_reservation = reservation.new_empty();+ let merged = StreamingMergeBuilder::new()+ .with_schema(spill_schema)+ .with_spill_manager(spill_manager)+ .with_sorted_spill_files(spills)+ .with_expressions(&spill_expr)+ .with_metrics(baseline_metrics.intermediate())+ .with_batch_size(batch_size)+ .with_reservation(merge_reservation)+ .build()?;+ let replay = OrderedFinalAggregateStream::new_with_input_and_metrics(+ &final_agg,+ &context,+ partition,+ merged,+ &InputOrderMode::Sorted,+ baseline_metrics.clone(),+ group_by_metrics,+ None,+ reservation,+ )?;+ Ok(Box::pin(replay))+ }+}++impl OrderedSingleAggregateStream {+ pub fn new(+ agg: &AggregateExec,+ context: &Arc<TaskContext>,+ partition: usize,+ ) -> Result<Self> {+ debug_assert!(matches!(+ agg.mode,+ AggregateMode::Single | AggregateMode::SinglePartitioned+ ));+ debug_assert_ne!(agg.input_order_mode, InputOrderMode::Linear);++ let schema = Arc::clone(&agg.schema);+ let input = agg.input.execute(partition, Arc::clone(context))?;+ let input_schema = input.schema();+ let batch_size = context.session_config().batch_size();+ let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);+ let spill_metrics = SpillMetrics::new(&agg.metrics, partition);+ let state_schema = Arc::new(create_schema(+ input_schema.as_ref(),+ &agg.group_by,+ &agg.aggr_expr,+ AggregateMode::Partial,+ )?);++ let table = OrderedAggregateTable::<SingleMarker>::new(+ agg,+ partition,+ Arc::clone(&schema),+ Arc::clone(&state_schema),+ batch_size,+ )?;++ let can_spill =+ matches!(agg.input_order_mode, InputOrderMode::PartiallySorted(_))+ && context.runtime_env().disk_manager.tmp_files_enabled();+ let spill_context = if can_spill {+ Some(Box::new(OrderedSingleSpillContext::new(+ agg,+ context,+ partition,+ batch_size,+ &agg.input_order_mode,+ &state_schema,+ spill_metrics,+ )?))+ } else {+ None+ };++ let reservation =+ MemoryConsumer::new(format!("OrderedSingleAggregateStream[{partition}]"))+ .with_can_spill(can_spill)+ .register(context.memory_pool());++ Ok(Self {+ schema,+ input,+ reservation,+ baseline_metrics,+ state: Some(OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context,+ }),+ })+ }++ fn close_input(&mut self) {+ let input_schema = self.input.schema();+ self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));+ }++ fn break_with_err(error: DataFusionError) -> OrderedSingleAggregateStateTransition {+ ControlFlow::Break((+ Poll::Ready(Some(Err(error))),+ OrderedSingleAggregateState::Error,+ ))+ }++ fn break_with_internal_err(message: &str) -> OrderedSingleAggregateStateTransition {+ Self::break_with_err(internal_datafusion_err!("{message}"))+ }++ /// Reserve memory for the current aggregate table.+ fn reservation_size_for_table(+ table: &OrderedAggregateTable<SingleMarker>,+ spill_context: Option<&OrderedSingleSpillContext>,+ ) -> usize {+ let table_size = table.memory_size();+ if spill_context.is_some() {+ // See `OrderedSingleAggregateStream` comments for how is it estimated+ table_size.saturating_add(table.num_groups().saturating_mul(size_of::<u32>()))+ } else {+ table_size+ }+ }++ /// Consumes one ordered raw input batch, then immediately emits+ /// finalized groups if the ordering proves any group is ready.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_reading_input(+ &mut self,+ cx: &mut Context<'_>,+ original_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let OrderedSingleAggregateState::ReadingInput {+ mut table,+ spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered single aggregate stream expected ReadingInput state",+ );+ };++ match self.input.poll_next_unpin(cx) {+ Poll::Pending => ControlFlow::Break((+ Poll::Pending,+ OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context,+ },+ )),+ Poll::Ready(Some(Ok(batch))) => {+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let result = table.aggregate_batch(&batch);+ timer.done();++ if let Err(e) = result {+ return Self::break_with_err(e);+ }++ // Check memory reservation, and potentially spill.+ let timer = elapsed_compute.timer();+ let resize_result =+ self.reservation+ .try_resize(Self::reservation_size_for_table(+ &table,+ spill_context.as_deref(),+ ));+ timer.done();+ match resize_result {+ Ok(()) => {}+ Err(e @ DataFusionError::ResourcesExhausted(_)) => {+ let Some(spill_context) = spill_context else {+ // `None` means spilling is not supported, see comments+ // at `OrderedSingleAggregateState` for details.+ return Self::break_with_err(e);+ };+ if table.is_empty() {+ return Self::break_with_internal_err(+ "Ordered single aggregate ran out of memory with no aggregated groups",+ );+ }+ return ControlFlow::Continue(+ OrderedSingleAggregateState::Spilling {+ table,+ spill_context,+ },+ );+ }+ Err(e) => {+ return Self::break_with_err(e);+ }+ }++ let result = if spill_context+ .as_ref()+ .is_some_and(|spill_context| spill_context.has_spills())+ {+ // Once one incomplete run is spilled, every remaining state+ // must participate in replay so no group is finalized twice.+ Ok(None)+ } else {+ let timer = elapsed_compute.timer();+ let result = table.next_output_batch();+ timer.done();+ result+ };++ match result {+ // Some finalized groups can be emitted. Yield them, then+ // continue aggregating input in the current state.+ Ok(Some(batch)) => {+ if let Err(e) =+ self.reservation+ .try_resize(Self::reservation_size_for_table(+ &table,+ spill_context.as_deref(),+ ))+ {+ return Self::break_with_err(e);+ }+ let next_state = OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context,+ };++ ControlFlow::Break((+ Poll::Ready(Some(Ok(+ batch.record_output(&self.baseline_metrics)+ ))),+ next_state,+ ))+ }+ // Can't do early emit, continue aggregating.+ Ok(None) => {+ ControlFlow::Continue(OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context,+ })+ }+ Err(e) => Self::break_with_err(e),+ }+ }+ Poll::Ready(Some(Err(e))) => Self::break_with_err(e),+ Poll::Ready(None) => {+ self.close_input();+ match spill_context {+ Some(spill_context) if spill_context.has_spills() => {+ ControlFlow::Continue(+ OrderedSingleAggregateState::PreparingMergeInput {+ table,+ spill_context,+ },+ )+ }+ _ => {+ table.input_done();+ ControlFlow::Continue(+ OrderedSingleAggregateState::ProducingOutput { table },+ )+ }+ }+ }+ }+ }++ /// Sorts and spills one complete in-memory state run, then resumes input.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_spilling(+ &mut self,+ original_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let OrderedSingleAggregateState::Spilling {+ mut table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered single aggregate stream expected Spilling state",+ );+ };++ // Sanity check: it's impossible to OOM when the table is empty+ if table.is_empty() {+ return Self::break_with_internal_err(+ "Ordered single aggregation entered Spilling with an empty table",+ );+ }++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let mut result = spill_context.spill_table(&mut table);++ // Spilling shrinks the aggregate table and releases its accumulated+ // memory. Update the reservation accordingly.+ if let Err(e) = self.reservation.try_resize(table.memory_size()) {+ result = Err(e);+ }++ timer.done();++ match result {+ // Finished spilling the aggregate table, continue aggregating from input+ Ok(()) => ControlFlow::Continue(OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context: Some(spill_context),+ }),+ Err(e) => Self::break_with_err(e),+ }+ }++ /// 1. Spills the last in-memory run.+ /// 2. Constructs a globally ordered input stream by applying a sort-preserving+ /// merge to all spills.+ /// 3. Constructs a replay stream: an ordered aggregate stream over the fully+ /// ordered input constructed from the spills.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_preparing_merge_input(+ &mut self,+ original_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let OrderedSingleAggregateState::PreparingMergeInput {+ mut table,+ mut spill_context,+ } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered single aggregate stream expected PreparingMergeInput state",+ );+ };++ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let replay = match spill_context.spill_table(&mut table) {+ Ok(()) => {+ let group_by_metrics = table.group_by_metrics();+ drop(table);+ match self.reservation.try_resize(0) {+ Ok(()) => (*spill_context).into_replay_stream(+ &self.baseline_metrics,+ group_by_metrics,+ self.reservation.new_empty(),+ ),+ Err(e) => Err(e),+ }+ }+ Err(e) => Err(e),+ };+ timer.done();++ match replay {+ Ok(stream) => {+ ControlFlow::Continue(OrderedSingleAggregateState::MergingSpills {+ stream,+ })+ }+ Err(e) => Self::break_with_err(e),+ }+ }++ /// Forwards output from the fully ordered stream that consumes the merged+ /// spill runs.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_merging_spills(+ &mut self,+ cx: &mut Context<'_>,+ original_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let OrderedSingleAggregateState::MergingSpills { mut stream } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered single aggregate stream expected MergingSpills state",+ );+ };++ match stream.poll_next_unpin(cx) {+ Poll::Pending => ControlFlow::Break((+ Poll::Pending,+ OrderedSingleAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Ok(batch))) => ControlFlow::Break((+ Poll::Ready(Some(Ok(batch))),+ OrderedSingleAggregateState::MergingSpills { stream },+ )),+ Poll::Ready(Some(Err(e))) => Self::break_with_err(e),+ Poll::Ready(None) => ControlFlow::Continue(OrderedSingleAggregateState::Done),+ }+ }++ /// Emits one batch after input is exhausted.+ ///+ /// `table.input_done()` has already made every remaining group safe to emit,+ /// so this state keeps draining until the table is empty.+ ///+ /// See comments at `poll_next()` for details.+ ///+ /// Returns the next operator state with control flow decision.+ fn handle_producing_output(+ &mut self,+ original_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let OrderedSingleAggregateState::ProducingOutput { table } = original_state+ else {+ return Self::break_with_internal_err(+ "Ordered single aggregate stream expected ProducingOutput state",+ );+ };++ let mut table = table;+ let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let result = table.next_output_batch();+ timer.done();++ match result {+ Ok(Some(batch)) => {+ let next_state = if table.is_empty() {+ drop(table);+ if let Err(e) = self.reservation.try_resize(0) {+ return Self::break_with_err(e);+ }+ OrderedSingleAggregateState::Done+ } else {+ if let Err(e) = self.reservation.try_resize(table.memory_size()) {+ return Self::break_with_err(e);+ }+ OrderedSingleAggregateState::ProducingOutput { table }+ };++ ControlFlow::Break((+ Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),+ next_state,+ ))+ }+ Err(e) => Self::break_with_err(e),+ Ok(None) => {+ drop(table);+ let next_state = OrderedSingleAggregateState::Done;+ if let Err(e) = self.reservation.try_resize(0) {+ return Self::break_with_err(e);+ }+ ControlFlow::Continue(next_state)+ }+ }+ }+}++impl Stream for OrderedSingleAggregateStream {+ type Item = Result<RecordBatch>;++ /// Entry point for the ordered single aggregate state machine.+ ///+ /// See comments in [`OrderedSingleAggregateStream`] for high-level ideas.+ ///+ /// State transition graph:+ ///+ /// ```text+ /// (start)+ /// -> ReadingInput+ /// The stream starts by polling ordered raw input and updating the+ /// ordered single aggregate table.+ ///+ /// ReadingInput+ /// -> ReadingInput+ /// Aggregate one input batch. If it fits in memory, optionally yield+ /// groups proven complete by the input ordering, then read the next batch.+ /// -> Spilling+ /// The table cannot reserve enough memory. Move all current states into+ /// one fully group-key-sorted spill run.+ /// -> ProducingOutput+ /// Input was exhausted without spilling. Mark every remaining group as+ /// complete and produce its final result.+ /// -> PreparingMergeInput+ /// Input was exhausted after spilling. Spill the last in-memory run and+ /// construct the ordered input used to merge all spill files.+ ///+ /// Spilling+ /// -> ReadingInput+ /// One sorted run was written; resume reading the original input.+ ///+ /// PreparingMergeInput+ /// Spill the final in-memory run and build the input ordered replay stream.+ /// -> MergingSpills+ /// The final run was spilled and the ordered replay stream was built.+ ///+ /// MergingSpills+ /// Aggregate the merged spill runs and emit final results.+ /// -> MergingSpills+ /// Forward one result batch from the fully ordered replay stream that+ /// consumes the sort-preserving merge.+ /// -> Done+ /// The merged spill input was fully aggregated.+ ///+ /// ProducingOutput+ /// -> ProducingOutput+ /// One remaining final aggregate batch was yielded; repeat to continue+ /// draining the table.+ /// -> Done+ /// All remaining groups were emitted.+ ///+ /// Any active state+ /// -> Error+ /// An error drops state-owned resources before it is returned.+ ///+ /// Error+ /// -> (end)+ ///+ /// Done+ /// -> (end)+ /// ```+ fn poll_next(+ mut self: std::pin::Pin<&mut Self>,+ cx: &mut Context<'_>,+ ) -> Poll<Option<Self::Item>> {+ loop {+ let cur_state = self+ .state+ .take()+ .expect("OrderedSingleAggregateStream state should not be None");++ let next_state = match cur_state {+ state @ OrderedSingleAggregateState::ReadingInput { .. } => {+ self.handle_reading_input(cx, state)+ }+ state @ OrderedSingleAggregateState::Spilling { .. } => {+ self.handle_spilling(state)+ }+ state @ OrderedSingleAggregateState::PreparingMergeInput { .. } => {+ self.handle_preparing_merge_input(state)+ }+ state @ OrderedSingleAggregateState::MergingSpills { .. } => {+ self.handle_merging_spills(cx, state)+ }+ state @ OrderedSingleAggregateState::ProducingOutput { .. } => {+ self.handle_producing_output(state)+ }+ state @ OrderedSingleAggregateState::Error => {+ self.close_input();+ self.reservation.free();+ self.state = Some(state);+ return Poll::Ready(None);+ }+ state @ OrderedSingleAggregateState::Done => {+ let _ = self.reservation.try_resize(0);+ self.state = Some(state);+ return Poll::Ready(None);+ }+ };++ match next_state {+ ControlFlow::Continue(next_state) => {+ self.state = Some(next_state);+ continue;+ }+ ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)) => {+ debug_assert!(matches!(+ next_state,+ OrderedSingleAggregateState::Error+ ));++ // The handler has already discarded its state-owned resources.+ // Release the remaining stream-owned resources before returning.+ self.close_input();+ self.reservation.free();+ self.state = Some(OrderedSingleAggregateState::Error);+ return Poll::Ready(Some(Err(e)));+ }+ ControlFlow::Break((poll, next_state)) => {+ self.state = Some(next_state);+ return poll;+ }+ }+ }+ }+}++impl RecordBatchStream for OrderedSingleAggregateStream {+ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }+}