Skip to content

Apache DataFusion SQL Query Engine

perf: materialize completed groups once in OrderedSingleAggregateStream (#25984)

Jay Zhan7c67e4cf185d
diff --git a/datafusion/physical-plan/benches/ordered_group_values.rs b/datafusion/physical-plan/benches/ordered_group_values.rs
index ae0e44a859..30af725cdd 100644
--- a/datafusion/physical-plan/benches/ordered_group_values.rs
+++ b/datafusion/physical-plan/benches/ordered_group_values.rs
@@ -19,7 +19,8 @@
//! existing streaming hash table on identical batches. Include emission and
//! cross-batch continuation, not just lookup. Input preparation is untimed.
//! The physical-plan benchmark includes aggregation but excludes SQL planning,
-//! sorting and I/O; the data already has a proven ordering.
+//! sorting and I/O; the data already has a proven ordering. A partially
+//! ordered case covers ordered prefixes that complete many groups at once.
use std::hint::black_box;
use std::sync::Arc;
@@ -186,6 +187,8 @@ fn check_case(schema: &SchemaRef, batches: &[Vec<ArrayRef>]) -> (usize, usize) {
fn aggregate_plan(
schema: &SchemaRef,
keys: Vec<Vec<ArrayRef>>,
+ sort_columns: &[&str],
+ input_order_mode: &InputOrderMode,
) -> Arc<dyn ExecutionPlan> {
let mut fields = schema.fields().to_vec();
fields.push(Arc::new(Field::new("v", DataType::Int64, false)));
@@ -197,10 +200,11 @@ fn aggregate_plan(
RecordBatch::try_new(Arc::clone(&schema), cols).unwrap()
})
.collect::<Vec<_>>();
- let ordering = LexOrdering::new(vec![
- PhysicalSortExpr::new_default(col("a", &schema).unwrap()),
- PhysicalSortExpr::new_default(col("b", &schema).unwrap()),
- ])
+ let ordering = LexOrdering::new(
+ sort_columns
+ .iter()
+ .map(|name| PhysicalSortExpr::new_default(col(name, &schema).unwrap())),
+ )
.unwrap();
let input =
TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap();
@@ -228,7 +232,7 @@ fn aggregate_plan(
schema,
)
.unwrap();
- assert_eq!(plan.input_order_mode(), &InputOrderMode::Sorted);
+ assert_eq!(plan.input_order_mode(), input_order_mode);
Arc::new(plan)
}
@@ -241,7 +245,8 @@ fn aggregation(c: &mut Criterion) {
for strings in [false, true] {
for run_length in [1, 8, 128, 8192] {
let (schema, keys) = inputs(run_length, 8192, strings);
- let plan = aggregate_plan(&schema, keys);
+ let plan =
+ aggregate_plan(&schema, keys, &["a", "b"], &InputOrderMode::Sorted);
let name =
format!("{}_run{run_length}", if strings { "string" } else { "int" });
group.bench_function(name, |b| {
@@ -261,5 +266,57 @@ fn aggregation(c: &mut Criterion) {
group.finish();
}
-criterion_group!(benches, grouping, aggregation);
+/// Input ordered by `a` only. Each `a` value has 32768 distinct `b` values, so
+/// every `a` boundary completes four default batches of groups at once.
+fn partially_ordered_aggregation(c: &mut Criterion) {
+ const GROUPS_PER_KEY: usize = 32768;
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let keys = (0..ROWS)
+ .step_by(8192)
+ .map(|start| {
+ let rows = start..(start + 8192).min(ROWS);
+ let a: ArrayRef = Arc::new(Int32Array::from_iter_values(
+ rows.clone().map(|row| (row / GROUPS_PER_KEY) as i32),
+ ));
+ let b: ArrayRef = Arc::new(Int32Array::from_iter_values(
+ rows.map(|row| (row % GROUPS_PER_KEY) as i32),
+ ));
+ vec![a, b]
+ })
+ .collect();
+ let plan = aggregate_plan(
+ &schema,
+ keys,
+ &["a"],
+ &InputOrderMode::PartiallySorted(vec![0]),
+ );
+ let runtime = Runtime::new().unwrap();
+ let mut group = c.benchmark_group("partially_ordered_aggregate_exec");
+ group.sample_size(10);
+ group.warm_up_time(Duration::from_millis(250));
+ group.measurement_time(Duration::from_secs(1));
+ group.bench_function("groups_per_key32768", |b| {
+ b.iter(|| {
+ black_box(
+ runtime
+ .block_on(collect(
+ Arc::clone(&plan),
+ Arc::new(TaskContext::default()),
+ ))
+ .unwrap(),
+ )
+ });
+ });
+ group.finish();
+}
+
+criterion_group!(
+ benches,
+ grouping,
+ aggregation,
+ partially_ordered_aggregation
+);
criterion_main!(benches);
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
index fd9e92635f..86c052009f 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
@@ -24,7 +24,6 @@ use std::sync::Arc;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
-use datafusion_common::assert_or_internal_err;
use datafusion_execution::memory_pool::proxy::VecAllocExt;
use datafusion_expr::{AggregateMetrics, EmitTo};
@@ -128,9 +127,6 @@ pub(in crate::aggregates) struct OrderedAggregateTable<OrderedAggrMode> {
/// to pass through or spill its current state.
pub(super) state_schema: SchemaRef,
- /// Maximum rows per emitted output batch, from config `batch_size`.
- pub(super) batch_size: usize,
-
/// Grouping and accumulator-specific timing metrics.
pub(super) group_by_metrics: GroupByMetrics,
@@ -188,17 +184,11 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {
input_schema: &SchemaRef,
output_schema: SchemaRef,
state_schema: SchemaRef,
- batch_size: usize,
input_order_mode: &InputOrderMode,
aggregate_mode: &AggregateMode,
filters: Vec<Option<Arc<dyn PhysicalExpr>>>,
metrics: OrderedAggregateTableMetrics,
) -> Result<Self> {
- assert_or_internal_err!(
- batch_size > 0,
- "OrderedAggregateTable requires config batch_size >= 1"
- );
-
let group_ordering = GroupOrdering::try_new(input_order_mode)?;
let group_schema = agg.group_by().group_schema(input_schema)?;
let group_values = new_group_values(group_schema, &group_ordering)?;
@@ -229,7 +219,6 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {
Ok(Self {
output_schema,
state_schema,
- batch_size,
group_by_metrics: metrics.group_by,
aggregate_argument_metrics: metrics.aggregate_arguments,
aggregate_accumulator_metrics: metrics.accumulator,
@@ -418,38 +407,8 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {
Ok(())
}
- /// Emits groups allowed by `GroupOrdering`, leaving only the current
- /// unfinished ordered-key range buffered.
- ///
- /// Each aggregation mode chooses a different `materialize_accumulator_fn`
- /// according to its semantics. For example, partial aggregation emits
- /// partial states to feed the final stage, so it uses
- /// [`datafusion_expr::GroupsAccumulator::state`].
- pub(super) fn next_output_batch_inner(
- &mut self,
- materialize_accumulator_fn: MaterializeAccumulatorFn,
- accumulator_phase: AccumulatorPhase,
- ) -> Result<Option<RecordBatch>> {
- if self.buffer.group_values.is_empty() {
- return Ok(None);
- }
-
- let Some(emit_to) = self.buffer.group_ordering.emit_to() else {
- return Ok(None);
- };
- let emit_to = match emit_to {
- EmitTo::First(n) => EmitTo::First(n.min(self.batch_size)),
- EmitTo::All if self.num_groups() > self.batch_size => {
- EmitTo::First(self.batch_size)
- }
- EmitTo::All => EmitTo::All,
- };
- self.materialize_groups(emit_to, materialize_accumulator_fn, accumulator_phase)
- .map(Some)
- }
-
/// Removes the selected groups once and materializes their output columns.
- /// The caller chooses the completed prefix and any output-size limit.
+ /// The caller chooses the completed prefix.
pub(super) fn materialize_groups(
&mut self,
emit_to: EmitTo,
@@ -459,11 +418,9 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {
let accumulator_metrics = Arc::clone(&self.aggregate_accumulator_metrics);
let output = self.group_by_metrics.time_emitting(|| {
let mut output = self.buffer.group_values.emit(emit_to)?;
- // EOF can also emit a prefix when a caller limits its batch size,
- // but the completed ordering state no longer tracks group indexes.
- if let EmitTo::First(n) = emit_to
- && matches!(self.buffer.group_ordering.emit_to(), Some(EmitTo::First(_)))
- {
+ // `EmitTo::All` is only used after `input_done`, when the ordering
+ // state no longer tracks group indexes.
+ if let EmitTo::First(n) = emit_to {
self.buffer.group_ordering.remove_groups(n);
}
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
index f70fc0bf0d..9cf6497e48 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
@@ -46,7 +46,6 @@ impl OrderedAggregateTable<FinalMarker> {
agg: &AggregateExec,
input_schema: &SchemaRef,
output_schema: SchemaRef,
- batch_size: usize,
input_order_mode: &InputOrderMode,
metrics: OrderedAggregateTableMetrics,
) -> Result<Self> {
@@ -55,7 +54,6 @@ impl OrderedAggregateTable<FinalMarker> {
input_schema,
output_schema,
Arc::clone(input_schema),
- batch_size,
input_order_mode,
&AggregateMode::Final,
vec![None; agg.aggr_expr().len()],
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
index 104855c5f0..3756c4b8e7 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
@@ -57,7 +57,6 @@ impl OrderedAggregateTable<PartialMarker> {
agg: &AggregateExec,
partition: usize,
output_schema: SchemaRef,
- batch_size: usize,
) -> Result<Self> {
let input_schema = agg.input().schema();
let state_schema = Arc::clone(&output_schema);
@@ -67,7 +66,6 @@ impl OrderedAggregateTable<PartialMarker> {
&input_schema,
output_schema,
state_schema,
- batch_size,
&agg.input_order_mode,
&AggregateMode::Partial,
agg.filter_expr().to_vec(),
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs
index dd266a68eb..89859bc036 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs
@@ -44,7 +44,6 @@ impl OrderedAggregateTable<SingleMarker> {
partition: usize,
output_schema: SchemaRef,
state_schema: SchemaRef,
- batch_size: usize,
) -> Result<Self> {
debug_assert!(matches!(
agg.mode,
@@ -58,7 +57,6 @@ impl OrderedAggregateTable<SingleMarker> {
&input_schema,
output_schema,
state_schema,
- batch_size,
&agg.input_order_mode,
&agg.mode,
agg.filter_expr().to_vec(),
@@ -80,14 +78,24 @@ impl OrderedAggregateTable<SingleMarker> {
)
}
- /// Emits the next batch of final aggregate values for groups proven complete
- /// by the input ordering.
- pub(in crate::aggregates) fn next_output_batch(
+ /// Materializes final results for all groups proven complete by the input
+ /// ordering, leaving the active ordered-key range in the table.
+ ///
+ /// Returns None if there are no completed groups.
+ pub(in crate::aggregates) fn take_completed_result_batch(
&mut self,
) -> Result<Option<RecordBatch>> {
- self.next_output_batch_inner(
+ if self.is_empty() {
+ return Ok(None);
+ }
+ let Some(emit_to) = self.group_ordering().emit_to() else {
+ return Ok(None);
+ };
+ self.materialize_groups(
+ emit_to,
HashAggregateAccumulator::evaluate_to_columns,
AccumulatorPhase::Evaluate,
)
+ .map(Some)
}
}
diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
index d9f5152456..daa5a85323 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs
@@ -189,7 +189,6 @@ impl OrderedFinalAggregateStream {
agg,
&input_schema,
Arc::clone(&schema),
- batch_size,
input_order_mode,
metrics,
)?;
diff --git a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
index 31046ec3d6..83a25620cd 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs
@@ -161,7 +161,6 @@ impl OrderedPartialAggregateStream {
agg,
partition,
Arc::clone(&schema),
- batch_size,
)?;
let reservation =
MemoryConsumer::new(format!("OrderedPartialAggregateStream[{partition}]"))
diff --git a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
index 834fae3cc6..57744dec57 100644
--- a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs
@@ -61,7 +61,12 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};
/// After each input batch, check whether any groups can be emitted eagerly to
/// improve memory efficiency. For example, if the last group key seen is
/// `k = 100`, it is safe to emit all groups with keys less than 100 because the
-/// input is ordered.
+/// input is ordered. Materialize that entire completed prefix once, then emit
+/// slices of it before reading more input. See
+/// [`OrderedPartialAggregateStream::into_stream`] for why this avoids
+/// repeatedly removing small batches of groups from the table.
+///
+/// [`OrderedPartialAggregateStream::into_stream`]: super::ordered_partial_stream::OrderedPartialAggregateStream::into_stream
///
/// # Memory Pressure and Spilling
///
@@ -98,6 +103,7 @@ pub(crate) struct OrderedSingleAggregateStream {
input: SendableRecordBatchStream,
reservation: MemoryReservation,
baseline_metrics: BaselineMetrics,
+ batch_size: usize,
state: Option<OrderedSingleAggregateState>,
}
@@ -114,8 +120,13 @@ enum OrderedSingleAggregateState {
table: OrderedAggregateTable<SingleMarker>,
spill_context: Box<AggregateSpill>,
},
- ProducingOutput {
- table: OrderedAggregateTable<SingleMarker>,
+ /// Emits one materialized batch in `batch_size` slices, then continues
+ /// with `next_state` (`ReadingInput`, or `Done` after input is exhausted).
+ Outputting {
+ batch: RecordBatch,
+ /// Reserved memory of `batch`, released when handing off the last slice.
+ batch_memory: usize,
+ next_state: Box<OrderedSingleAggregateState>,
},
PreparingMergeInput {
table: OrderedAggregateTable<SingleMarker>,
@@ -167,7 +178,6 @@ impl OrderedSingleAggregateStream {
partition,
Arc::clone(&schema),
Arc::clone(&state_schema),
- batch_size,
)?;
let can_spill =
@@ -201,6 +211,7 @@ impl OrderedSingleAggregateStream {
input,
reservation,
baseline_metrics,
+ batch_size,
state: Some(OrderedSingleAggregateState::ReadingInput {
table,
spill_context,
@@ -238,8 +249,38 @@ impl OrderedSingleAggregateStream {
}
}
- /// Consumes one ordered raw input batch, then immediately emits
- /// finalized groups if the ordering proves any group is ready.
+ /// Reserves `batch` on top of `table_memory` and moves to `Outputting`,
+ /// which emits it in `batch_size` slices before continuing with
+ /// `next_state`. If the batch cannot be reserved, hands it off whole.
+ fn start_outputting(
+ &mut self,
+ batch: RecordBatch,
+ table_memory: usize,
+ next_state: OrderedSingleAggregateState,
+ ) -> OrderedSingleAggregateStateTransition {
+ let batch_memory = batch.get_array_memory_size();
+ match self.reservation.try_resize(table_memory + batch_memory) {
+ Ok(()) => ControlFlow::Continue(OrderedSingleAggregateState::Outputting {
+ batch,
+ batch_memory,
+ next_state: Box::new(next_state),
+ }),
+ Err(DataFusionError::ResourcesExhausted(_)) => {
+ // Only the retained table needs to remain reserved.
+ if let Err(e) = self.reservation.try_resize(table_memory) {
+ return Self::break_with_err(e);
+ }
+ ControlFlow::Break((
+ Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),
+ next_state,
+ ))
+ }
+ Err(e) => Self::break_with_err(e),
+ }
+ }
+
+ /// Consumes one ordered raw input batch, then materializes all finalized
+ /// groups if the ordering proves any group is ready.
///
/// See comments at `poll_next()` for details.
///
@@ -320,35 +361,27 @@ impl OrderedSingleAggregateStream {
Ok(None)
} else {
let timer = elapsed_compute.timer();
- let result = table.next_output_batch();
+ let result = table.take_completed_result_batch();
timer.done();
result
};
match result {
- // Some finalized groups can be emitted. Yield them, then
- // continue aggregating input in the current state.
+ // Some finalized groups can be emitted. Yield them in
+ // slices, then continue aggregating input.
Ok(Some(batch)) => {
- if let Err(e) =
- self.reservation
- .try_resize(Self::reservation_size_for_table(
- &table,
- spill_context.as_deref(),
- ))
- {
- return Self::break_with_err(e);
- }
- let next_state = OrderedSingleAggregateState::ReadingInput {
- table,
- spill_context,
- };
-
- ControlFlow::Break((
- Poll::Ready(Some(Ok(
- batch.record_output(&self.baseline_metrics)
- ))),
- next_state,
- ))
+ let table_memory = Self::reservation_size_for_table(
+ &table,
+ spill_context.as_deref(),
+ );
+ self.start_outputting(
+ batch,
+ table_memory,
+ OrderedSingleAggregateState::ReadingInput {
+ table,
+ spill_context,
+ },
+ )
}
// Can't do early emit, continue aggregating.
Ok(None) => {
@@ -374,9 +407,23 @@ impl OrderedSingleAggregateStream {
}
_ => {
table.input_done();
- ControlFlow::Continue(
- OrderedSingleAggregateState::ProducingOutput { table },
- )
+ let elapsed_compute =
+ self.baseline_metrics.elapsed_compute().clone();
+ let timer = elapsed_compute.timer();
+ let result = table.take_completed_result_batch();
+ drop(table);
+ timer.done();
+ match result {
+ Ok(Some(batch)) => self.start_outputting(
+ batch,
+ 0,
+ OrderedSingleAggregateState::Done,
+ ),
+ Ok(None) => {
+ ControlFlow::Continue(OrderedSingleAggregateState::Done)
+ }
+ Err(e) => Self::break_with_err(e),
+ }
}
}
}
@@ -520,61 +567,47 @@ impl OrderedSingleAggregateStream {
}
}
- /// Emits one batch after input is exhausted.
- ///
- /// `table.input_done()` has already made every remaining group safe to emit,
- /// so this state keeps draining until the table is empty.
+ /// Emits the next `batch_size` slice of a materialized batch.
///
/// See comments at `poll_next()` for details.
///
/// Returns the next operator state with control flow decision.
- fn handle_producing_output(
+ fn handle_outputting(
&mut self,
original_state: OrderedSingleAggregateState,
) -> OrderedSingleAggregateStateTransition {
- let OrderedSingleAggregateState::ProducingOutput { table } = original_state
+ let OrderedSingleAggregateState::Outputting {
+ batch,
+ batch_memory,
+ next_state,
+ } = original_state
else {
return Self::break_with_internal_err(
- "Ordered single aggregate stream expected ProducingOutput state",
+ "Ordered single aggregate stream expected Outputting state",
);
};
- let mut table = table;
- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();
- let timer = elapsed_compute.timer();
- let result = table.next_output_batch();
- timer.done();
-
- match result {
- Ok(Some(batch)) => {
- let next_state = if table.is_empty() {
- drop(table);
- if let Err(e) = self.reservation.try_resize(0) {
- return Self::break_with_err(e);
- }
- OrderedSingleAggregateState::Done
- } else {
- if let Err(e) = self.reservation.try_resize(table.memory_size()) {
- return Self::break_with_err(e);
- }
- OrderedSingleAggregateState::ProducingOutput { table }
- };
-
- ControlFlow::Break((
- Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),
+ if batch.num_rows() > self.batch_size {
+ let output = batch.slice(0, self.batch_size);
+ let batch = batch.slice(self.batch_size, batch.num_rows() - self.batch_size);
+ return ControlFlow::Break((
+ Poll::Ready(Some(Ok(output.record_output(&self.baseline_metrics)))),
+ OrderedSingleAggregateState::Outputting {
+ batch,
+ batch_memory,
next_state,
- ))
- }
- Err(e) => Self::break_with_err(e),
- Ok(None) => {
- drop(table);
- let next_state = OrderedSingleAggregateState::Done;
- if let Err(e) = self.reservation.try_resize(0) {
- return Self::break_with_err(e);
- }
- ControlFlow::Continue(next_state)
- }
+ },
+ ));
}
+
+ // The final slice transfers ownership of the buffers to the consumer.
+ if let Err(e) = self.reservation.try_shrink(batch_memory) {
+ return Self::break_with_err(e);
+ }
+ ControlFlow::Break((
+ Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),
+ *next_state,
+ ))
}
}
@@ -595,14 +628,18 @@ impl Stream for OrderedSingleAggregateStream {
///
/// ReadingInput
/// -> ReadingInput
- /// Aggregate one input batch. If it fits in memory, optionally yield
- /// groups proven complete by the input ordering, then read the next batch.
+ /// Aggregate one input batch. If it fits in memory and no groups are
+ /// complete, read the next batch.
/// -> Spilling
/// The table cannot reserve enough memory. Move all current states into
/// one fully group-key-sorted spill run.
- /// -> ProducingOutput
- /// Input was exhausted without spilling. Mark every remaining group as
- /// complete and produce its final result.
+ /// -> Outputting
+ /// Either the input ordering proves some groups complete, or input was
+ /// exhausted without spilling and every remaining group is complete.
+ /// Materialize all of them once into one batch. If the batch cannot be
+ /// reserved, yield it whole and go to the state after `Outputting`.
+ /// -> Done
+ /// Input was exhausted without spilling and no groups remain.
/// -> PreparingMergeInput
/// Input was exhausted after spilling. Spill the last in-memory run and
/// construct the ordered input used to merge all spill files.
@@ -624,12 +661,15 @@ impl Stream for OrderedSingleAggregateStream {
/// -> Done
/// The merged spill input was fully aggregated.
///
- /// ProducingOutput
- /// -> ProducingOutput
- /// One remaining final aggregate batch was yielded; repeat to continue
- /// draining the table.
+ /// Outputting
+ /// Yield one `batch_size` slice of the materialized batch, keeping the
+ /// batch reserved until its last slice is handed off.
+ /// -> Outputting
+ /// More slices remain.
+ /// -> ReadingInput
+ /// The batch was fully emitted; resume aggregating input.
/// -> Done
- /// All remaining groups were emitted.
+ /// The batch was fully emitted after input was exhausted.
///
/// Any active state
/// -> Error
@@ -664,8 +704,8 @@ impl Stream for OrderedSingleAggregateStream {
state @ OrderedSingleAggregateState::MergingSpills { .. } => {
self.handle_merging_spills(cx, state)
}
- state @ OrderedSingleAggregateState::ProducingOutput { .. } => {
- self.handle_producing_output(state)
+ state @ OrderedSingleAggregateState::Outputting { .. } => {
+ self.handle_outputting(state)
}
state @ OrderedSingleAggregateState::Error => {
self.close_input();
@@ -711,3 +751,98 @@ impl RecordBatchStream for OrderedSingleAggregateStream {
Arc::clone(&self.schema)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::aggregates::PhysicalGroupBy;
+ use crate::collect;
+ use crate::test::TestMemoryExec;
+ use arrow::array::Int64Array;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::test_util::batches_to_sort_string;
+ use datafusion_execution::config::SessionConfig;
+ use datafusion_execution::memory_pool::{
+ GreedyMemoryPool, MemoryPool, PeakRecordingPool,
+ };
+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+ use datafusion_functions_aggregate::sum::sum_udaf;
+ use datafusion_physical_expr::PhysicalSortExpr;
+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+ use datafusion_physical_expr::expressions::col;
+ use datafusion_physical_expr_common::sort_expr::LexOrdering;
+
+ /// Runs `SELECT a, b, SUM(v) GROUP BY a, b` with `batch_size = 4` over
+ /// input ordered by `a`. Each `a` boundary completes 43 groups at once.
+ async fn run(pool: Arc<dyn MemoryPool>) -> Result<Vec<RecordBatch>> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int64, false),
+ Field::new("b", DataType::Int64, false),
+ Field::new("v", DataType::Int64, false),
+ ]));
+ let batches = [0, 0, 1, 1, 2, 2]
+ .map(|a| {
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int64Array::from(vec![a; 43])),
+ Arc::new(Int64Array::from_iter_values(0..43)),
+ Arc::new(Int64Array::from_iter_values(0..43)),
+ ],
+ )
+ .unwrap()
+ })
+ .to_vec();
+ let input = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None)?
+ .try_with_sort_information(vec![
+ LexOrdering::new([PhysicalSortExpr::new_default(col("a", &schema)?)])
+ .unwrap(),
+ ])?;
+ let aggregate = AggregateExec::try_new(
+ AggregateMode::Single,
+ PhysicalGroupBy::new_single(vec![
+ (col("a", &schema)?, "a".into()),
+ (col("b", &schema)?, "b".into()),
+ ]),
+ vec![Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("sum")
+ .build()?,
+ )],
+ vec![None],
+ Arc::new(TestMemoryExec::update_cache(&Arc::new(input))),
+ Arc::clone(&schema),
+ )?;
+ let context = TaskContext::default()
+ .with_session_config(
+ SessionConfig::new()
+ .with_batch_size(4)
+ .set_bool("datafusion.execution.enable_migration_aggregate", true),
+ )
+ .with_runtime(
+ RuntimeEnvBuilder::new()
+ .with_memory_pool(pool)
+ .build_arc()?,
+ );
+ collect(Arc::new(aggregate), Arc::new(context)).await
+ }
+
+ /// One byte below the unlimited peak, the completed groups cannot be
+ /// reserved next to the table, so they are handed off without slicing.
+ /// The result must not change.
+ #[tokio::test]
+ async fn completed_groups_under_memory_pressure() -> Result<()> {
+ let pool = Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new(
+ usize::MAX,
+ ))));
+ let expected = run(Arc::clone(&pool) as _).await?;
+ let limit = pool.peak_reserved() - 1;
+ let actual = run(Arc::new(GreedyMemoryPool::new(limit))).await?;
+ assert_eq!(
+ batches_to_sort_string(&actual),
+ batches_to_sort_string(&expected)
+ );
+ Ok(())
+ }
+}