Skip to content

Apache DataFusion SQL Query Engine

Avoid repeated `EmitTo::First` in partial hash aggregate output (#23250)

Huang Qiwei7d9f6ea405d5
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
index 719fbe93e5..a0d8204180 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -182,7 +182,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
acc + state.group_values.size()
+ state.batch_group_indices.allocated_size()
}
- AggregateHashTableState::OutputtingMaterializedFinal(output) => {
+ AggregateHashTableState::OutputtingMaterialized(output) => {
output.memory_size()
}
AggregateHashTableState::Done => 0,
@@ -214,15 +214,6 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
}
}
-pub(super) fn emit_to_for_batch_size(batch_size: usize, group_count: usize) -> EmitTo {
- debug_assert!(batch_size > 0);
- if group_count <= batch_size {
- EmitTo::All
- } else {
- EmitTo::First(batch_size)
- }
-}
-
/// State and argument information for a single Aggregate
///
/// For example, for `SELECT COUNT(x), SUM(y WHERE z > 10) ...` there would be two
@@ -304,24 +295,25 @@ pub(super) enum AggregateHashTableState {
Building(AggregateHashTableBuffer),
/// Emitting results directly from group keys and aggregate state.
Outputting(AggregateHashTableBuffer),
- /// Materialize all the output results, and then incrementally output in the `OutputtingMaterializedFinal` state.
+ /// Materialize all the output results, and then incrementally output in the `OutputtingMaterialized` state.
///
/// Note this is a temporary solution until the `GroupValues` issue is solved:
/// Issue: <https://github.com/apache/datafusion/issues/23178>
- OutputtingMaterializedFinal(MaterializedFinalOutput),
+ OutputtingMaterialized(MaterializedAggregateOutput),
Done,
}
-/// Fully evaluated final aggregate output and the next row offset to emit.
+/// Fully evaluated aggregate output and the next row offset to emit.
///
-/// Final aggregate evaluation consumes accumulator state, so final output is
-/// materialized once and then sliced to honor `batch_size` across output polls.
-pub(super) struct MaterializedFinalOutput {
+/// Final aggregate evaluation consumes accumulator state, and partial terminal
+/// output should not repeatedly renumber group values with `EmitTo::First`.
+/// Materialize once and then slice to honor `batch_size` across output polls.
+pub(super) struct MaterializedAggregateOutput {
batch: RecordBatch,
offset: usize,
}
-impl MaterializedFinalOutput {
+impl MaterializedAggregateOutput {
pub(super) fn new(batch: RecordBatch) -> Self {
Self { batch, offset: 0 }
}
@@ -496,7 +488,7 @@ mod tests {
use super::*;
#[test]
- fn materialized_final_output_slices_batches_until_exhausted() -> Result<()> {
+ fn materialized_aggregate_output_slices_batches_until_exhausted() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new(
"group_col",
DataType::Int32,
@@ -506,7 +498,7 @@ mod tests {
schema,
vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
)?;
- let mut output = MaterializedFinalOutput::new(batch);
+ let mut output = MaterializedAggregateOutput::new(batch);
assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![1, 2]);
assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![3, 4]);
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
index c3e4f831c4..bd70d10858 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
@@ -26,7 +26,7 @@ use crate::aggregates::AggregateExec;
use super::common::{
AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState, FinalMarker,
- MaterializedFinalOutput,
+ MaterializedAggregateOutput,
};
/// Methods specific to the aggregate hash table used in the final aggregation stage.
@@ -57,8 +57,8 @@ impl AggregateHashTable<FinalMarker> {
) -> Result<Option<RecordBatch>> {
let output_schema = Arc::clone(&self.output_schema);
let batch_size = self.batch_size;
- // Take ownership of the output state. Note `emit_next_materialized_batch`
- // updates state after it emits a materialized slice.
+ // Take ownership of the output state. `emit_next_materialized_batch`
+ // restores `self.state` to `OutputtingMaterialized` or `Done`.
match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {
AggregateHashTableState::Outputting(state) => {
if state.group_values.is_empty() {
@@ -68,7 +68,7 @@ impl AggregateHashTable<FinalMarker> {
let output = self.materialize_final_output(state, output_schema)?;
Ok(self.emit_next_materialized_batch(output, batch_size))
}
- AggregateHashTableState::OutputtingMaterializedFinal(output) => {
+ AggregateHashTableState::OutputtingMaterialized(output) => {
Ok(self.emit_next_materialized_batch(output, batch_size))
}
AggregateHashTableState::Done => Ok(None),
@@ -82,7 +82,7 @@ impl AggregateHashTable<FinalMarker> {
&self,
mut state: AggregateHashTableBuffer,
output_schema: SchemaRef,
- ) -> Result<MaterializedFinalOutput> {
+ ) -> Result<MaterializedAggregateOutput> {
// Final aggregate evaluation consumes accumulator state. Evaluate all
// groups once, then slice the materialized batch on subsequent polls.
let emit_to = EmitTo::All;
@@ -96,19 +96,19 @@ impl AggregateHashTable<FinalMarker> {
let batch = RecordBatch::try_new(output_schema, output)?;
debug_assert!(batch.num_rows() > 0);
- Ok(MaterializedFinalOutput::new(batch))
+ Ok(MaterializedAggregateOutput::new(batch))
}
fn emit_next_materialized_batch(
&mut self,
- mut output: MaterializedFinalOutput,
+ mut output: MaterializedAggregateOutput,
batch_size: usize,
) -> Option<RecordBatch> {
let batch = output.next_batch(batch_size);
if output.is_exhausted() {
self.state = AggregateHashTableState::Done;
} else {
- self.state = AggregateHashTableState::OutputtingMaterializedFinal(output);
+ self.state = AggregateHashTableState::OutputtingMaterialized(output);
}
batch
}
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
index 9d226aa28b..53b44f5d9a 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
@@ -23,6 +23,7 @@ use arrow::array::{ArrayRef, BooleanArray, new_null_array};
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
+use datafusion_expr::EmitTo;
use crate::aggregates::group_values::new_group_values;
use crate::aggregates::order::GroupOrdering;
@@ -30,8 +31,8 @@ use crate::aggregates::{AggregateExec, group_id_array, max_duplicate_ordinal};
use super::common::{
AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState,
- EvaluatedAccumulatorArgs, HashAggregateAccumulator, PartialMarker, PartialSkipMarker,
- emit_to_for_batch_size,
+ EvaluatedAccumulatorArgs, HashAggregateAccumulator, MaterializedAggregateOutput,
+ PartialMarker, PartialSkipMarker,
};
/// Methods specific to the aggregate hash table used in the partial aggregation stage.
@@ -62,43 +63,60 @@ impl AggregateHashTable<PartialMarker> {
) -> Result<Option<RecordBatch>> {
let output_schema = Arc::clone(&self.output_schema);
let batch_size = self.batch_size;
- match &mut self.state {
+ // Take ownership of the output state. `emit_next_materialized_batch`
+ // restores `self.state` to `OutputtingMaterialized` or `Done`.
+ match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {
AggregateHashTableState::Outputting(state) => {
if state.group_values.is_empty() {
- self.state = AggregateHashTableState::Done;
return Ok(None);
}
- let emit_to =
- emit_to_for_batch_size(batch_size, state.group_values.len());
- let timer = self.group_by_metrics.emitting_time.timer();
- let mut output = state.group_values.emit(emit_to)?;
-
- for acc in state.accumulators.iter_mut() {
- output.extend(acc.state(emit_to)?);
- }
- let done = state.group_values.is_empty();
- drop(timer);
-
- let batch = RecordBatch::try_new(output_schema, output)?;
- debug_assert!(batch.num_rows() > 0);
- if done {
- self.state = AggregateHashTableState::Done;
- }
- Ok(Some(batch))
+ let output = self.materialize_partial_output(state, output_schema)?;
+ Ok(self.emit_next_materialized_batch(output, batch_size))
+ }
+ AggregateHashTableState::OutputtingMaterialized(output) => {
+ Ok(self.emit_next_materialized_batch(output, batch_size))
}
AggregateHashTableState::Done => Ok(None),
AggregateHashTableState::Building(_) => {
internal_err!("next_output_batch must be called in the outputting state")
}
- AggregateHashTableState::OutputtingMaterializedFinal(_) => {
- internal_err!(
- "partial aggregate output should not materialize final output"
- )
- }
}
}
+ fn materialize_partial_output(
+ &self,
+ mut state: AggregateHashTableBuffer,
+ output_schema: SchemaRef,
+ ) -> Result<MaterializedAggregateOutput> {
+ let emit_to = EmitTo::All;
+ let timer = self.group_by_metrics.emitting_time.timer();
+ let mut output = state.group_values.emit(emit_to)?;
+
+ for acc in state.accumulators.iter_mut() {
+ output.extend(acc.state(emit_to)?);
+ }
+ drop(timer);
+
+ let batch = RecordBatch::try_new(output_schema, output)?;
+ debug_assert!(batch.num_rows() > 0);
+ Ok(MaterializedAggregateOutput::new(batch))
+ }
+
+ fn emit_next_materialized_batch(
+ &mut self,
+ mut output: MaterializedAggregateOutput,
+ batch_size: usize,
+ ) -> Option<RecordBatch> {
+ let batch = output.next_batch(batch_size);
+ if output.is_exhausted() {
+ self.state = AggregateHashTableState::Done;
+ } else {
+ self.state = AggregateHashTableState::OutputtingMaterialized(output);
+ }
+ batch
+ }
+
pub(in crate::aggregates) fn can_skip_aggregation(&self) -> bool {
self.state
.building()
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 4f5b893578..07c77c860e 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -2434,8 +2434,8 @@ mod tests {
};
use arrow::array::{
- DictionaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StructArray,
- UInt32Array, UInt64Array,
+ BooleanArray, DictionaryArray, Float32Array, Float64Array, Int32Array,
+ Int64Array, StructArray, UInt32Array, UInt64Array,
};
use arrow::compute::{SortOptions, concat_batches};
use arrow::datatypes::Int32Type;
@@ -2445,7 +2445,10 @@ mod tests {
use datafusion_execution::memory_pool::FairSpillPool;
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
- use datafusion_expr::{AggregateUDF, AggregateUDFImpl, Signature, Volatility};
+ use datafusion_expr::{
+ Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator,
+ Signature, Volatility,
+ };
use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;
use datafusion_functions_aggregate::array_agg::array_agg_udaf;
use datafusion_functions_aggregate::average::avg_udaf;
@@ -3229,6 +3232,76 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn partial_grouped_aggregate_materializes_before_slicing() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("value", DataType::Int32, false),
+ ]));
+ let input_batches = vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3])),
+ Arc::new(Int32Array::from(vec![10, 20, 30])),
+ ],
+ )?];
+ let input =
+ TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
+ let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));
+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
+ AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("no_first_emit(value)")
+ .build()?,
+ )];
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ group_by,
+ aggregates,
+ vec![None],
+ input,
+ Arc::clone(&schema),
+ )?);
+ let task_ctx = Arc::new(
+ TaskContext::default().with_session_config(
+ SessionConfig::new()
+ .with_batch_size(2)
+ .set_bool("datafusion.execution.enable_migration_aggregate", true)
+ .set(
+ "datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
+ &ScalarValue::Float64(Some(2.0)),
+ ),
+ ),
+ );
+
+ let stream = aggregate.execute_typed(0, &task_ctx)?;
+ assert!(matches!(stream, StreamType::PartialHash(_)));
+
+ let stream: SendableRecordBatchStream = stream.into();
+ let batches = collect(stream).await?;
+ assert_eq!(
+ batches
+ .iter()
+ .map(RecordBatch::num_rows)
+ .collect::<Vec<_>>(),
+ vec![2, 1]
+ );
+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +-----+-----------------------------+
+ | key | no_first_emit(value)[count] |
+ +-----+-----------------------------+
+ | 1 | 1 |
+ | 2 | 1 |
+ | 3 | 1 |
+ +-----+-----------------------------+
+ ");
+
+ Ok(())
+ }
+
#[tokio::test]
async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {
let schema =
@@ -6256,6 +6329,140 @@ mod tests {
}
}
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct NoFirstEmitUdaf {
+ signature: Signature,
+ }
+
+ impl NoFirstEmitUdaf {
+ fn new() -> Self {
+ Self {
+ signature: Signature::exact(vec![DataType::Int32], Volatility::Immutable),
+ }
+ }
+ }
+
+ impl AggregateUDFImpl for NoFirstEmitUdaf {
+ fn name(&self) -> &str {
+ "no_first_emit"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Int64)
+ }
+
+ fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
+ Ok(vec![Arc::new(Field::new(
+ format!("{}[count]", args.name),
+ DataType::Int64,
+ false,
+ ))])
+ }
+
+ fn accumulator(
+ &self,
+ _acc_args: AccumulatorArgs,
+ ) -> Result<Box<dyn Accumulator>> {
+ Ok(Box::new(NoFirstEmitAccumulator))
+ }
+
+ fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {
+ true
+ }
+
+ fn create_groups_accumulator(
+ &self,
+ _args: AccumulatorArgs,
+ ) -> Result<Box<dyn GroupsAccumulator>> {
+ Ok(Box::new(NoFirstEmitGroupsAccumulator { counts: vec![] }))
+ }
+ }
+
+ #[derive(Debug)]
+ struct NoFirstEmitAccumulator;
+
+ impl Accumulator for NoFirstEmitAccumulator {
+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+
+ fn evaluate(&mut self) -> Result<ScalarValue> {
+ Ok(ScalarValue::Int64(Some(0)))
+ }
+
+ fn size(&self) -> usize {
+ size_of_val(self)
+ }
+
+ fn state(&mut self) -> Result<Vec<ScalarValue>> {
+ Ok(vec![ScalarValue::Int64(Some(0))])
+ }
+
+ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {
+ Ok(())
+ }
+ }
+
+ #[derive(Debug)]
+ struct NoFirstEmitGroupsAccumulator {
+ counts: Vec<i64>,
+ }
+
+ impl NoFirstEmitGroupsAccumulator {
+ fn emit_counts(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
+ match emit_to {
+ EmitTo::All => {
+ let counts = std::mem::take(&mut self.counts);
+ Ok(Arc::new(Int64Array::from(counts)))
+ }
+ EmitTo::First(_) => internal_err!(
+ "partial grouped aggregate output must materialize with EmitTo::All before slicing"
+ ),
+ }
+ }
+ }
+
+ impl GroupsAccumulator for NoFirstEmitGroupsAccumulator {
+ fn update_batch(
+ &mut self,
+ _values: &[ArrayRef],
+ group_indices: &[usize],
+ _opt_filter: Option<&BooleanArray>,
+ total_num_groups: usize,
+ ) -> Result<()> {
+ self.counts.resize(total_num_groups, 0);
+ for group_index in group_indices {
+ self.counts[*group_index] += 1;
+ }
+ Ok(())
+ }
+
+ fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
+ self.emit_counts(emit_to)
+ }
+
+ fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
+ Ok(vec![self.emit_counts(emit_to)?])
+ }
+
+ fn merge_batch(
+ &mut self,
+ _values: &[ArrayRef],
+ _group_indices: &[usize],
+ _total_num_groups: usize,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn size(&self) -> usize {
+ size_of_val(self) + self.counts.capacity() * size_of::<i64>()
+ }
+ }
+
/// Test that [`AggregateExec::with_dynamic_filter_expr`] overrides the existing dynamic filter
#[test]
fn test_with_dynamic_filter() -> Result<()> {