Skip to content

Apache DataFusion SQL Query Engine

minor: move `create_group_accumulator` out of the legacy aggregate stream (#25539)

Jay Zhan9bffbea2a4cc
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
index f517f6a998..0c5c4185e2 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -29,14 +29,15 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::{Result, internal_err};
use datafusion_execution::memory_pool::proxy::VecAllocExt;
use datafusion_expr::{AggregateMetrics, EmitTo, GroupsAccumulator};
+use datafusion_physical_expr::GroupsAccumulatorAdapter;
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use log::debug;
use crate::PhysicalExpr;
use crate::aggregates::group_values::{
AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,
GroupByMetrics, GroupValues, new_group_values,
};
-use crate::aggregates::grouped_hash_stream::create_group_accumulator;
use crate::aggregates::order::GroupOrdering;
use crate::aggregates::{
AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
@@ -56,6 +57,29 @@ pub(in crate::aggregates) struct PartialSkipMarker;
/// Marker for partial state -> final value aggregation.
pub(in crate::aggregates) struct FinalMarker;
+/// Create an accumulator for `agg_expr` -- a [`GroupsAccumulator`] if
+/// that is supported by the aggregate, or a
+/// [`GroupsAccumulatorAdapter`] if not.
+pub(in crate::aggregates) fn create_group_accumulator(
+ agg_expr: &Arc<AggregateFunctionExpr>,
+ metrics: Arc<dyn AggregateMetrics>,
+) -> Result<Box<dyn GroupsAccumulator>> {
+ if agg_expr.groups_accumulator_supported() {
+ agg_expr.create_groups_accumulator_with_metrics(metrics)
+ } else {
+ // Note in the log when the slow path is used
+ debug!(
+ "Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
+ agg_expr.name()
+ );
+ let agg_expr = Arc::clone(agg_expr);
+ let mut adapter =
+ GroupsAccumulatorAdapter::new(move || agg_expr.create_accumulator());
+ adapter.set_metrics(metrics);
+ Ok(Box::new(adapter))
+ }
+}
+
/// Grouped hash table shared by the partial and final paths.
///
/// While building, it consumes input batches and updates group / accumulator
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
index 6446c2ecc1..26a1beb0a2 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
@@ -34,7 +34,6 @@ use crate::aggregates::group_values::{
AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,
GroupByMetrics, GroupValues, new_group_values,
};
-use crate::aggregates::grouped_hash_stream::create_group_accumulator;
use crate::aggregates::order::GroupOrdering;
use crate::aggregates::{
AggregateExec, AggregateMode, PhysicalGroupBy, aggregate_expressions,
@@ -44,7 +43,7 @@ use crate::aggregates::{
use super::AggregateTableMetrics;
use super::common::{
AggregateAccumulator, AggregateBatchFn, AggregateHashTable, EvaluatedAggregateBatch,
- MaterializeAccumulatorFn,
+ MaterializeAccumulatorFn, create_group_accumulator,
};
#[derive(Clone)]
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs
index 29ef4a662f..005b2bac09 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs
@@ -99,7 +99,7 @@ impl AggregateTableMetrics {
pub(super) use common::{
AggregateHashTable, FinalMarker, PartialMarker, PartialReduceMarker,
- PartialSkipMarker, SingleMarker,
+ PartialSkipMarker, SingleMarker, create_group_accumulator,
};
pub(super) use common_ordered::{OrderedAggregateTable, OrderedAggregateTableMetrics};
diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
index 092a1b26b3..042f4a9044 100644
--- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
@@ -34,7 +34,7 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use std::vec;
-use super::aggregate_hash_table::accumulator_phases;
+use super::aggregate_hash_table::{accumulator_phases, create_group_accumulator};
use super::order::GroupOrdering;
use super::skip_partial::SkipAggregationProbe;
use super::{AggregateExec, format_human_display};
@@ -64,10 +64,9 @@ use datafusion_common::{
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::proxy::VecAllocExt;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
-use datafusion_expr::{AggregateMetrics, EmitTo, GroupsAccumulator};
-use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use datafusion_expr::{EmitTo, GroupsAccumulator};
+use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_expr::expressions::Column;
-use datafusion_physical_expr::{GroupsAccumulatorAdapter, PhysicalSortExpr};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;
@@ -652,29 +651,6 @@ impl GroupedHashAggregateStream {
}
}
-/// Create an accumulator for `agg_expr` -- a [`GroupsAccumulator`] if
-/// that is supported by the aggregate, or a
-/// [`GroupsAccumulatorAdapter`] if not.
-pub(crate) fn create_group_accumulator(
- agg_expr: &Arc<AggregateFunctionExpr>,
- metrics: Arc<dyn AggregateMetrics>,
-) -> Result<Box<dyn GroupsAccumulator>> {
- if agg_expr.groups_accumulator_supported() {
- agg_expr.create_groups_accumulator_with_metrics(metrics)
- } else {
- // Note in the log when the slow path is used
- debug!(
- "Creating GroupsAccumulatorAdapter for {}: {agg_expr:?}",
- agg_expr.name()
- );
- let agg_expr = Arc::clone(agg_expr);
- let mut adapter =
- GroupsAccumulatorAdapter::new(move || agg_expr.create_accumulator());
- adapter.set_metrics(metrics);
- Ok(Box::new(adapter))
- }
-}
-
impl Stream for GroupedHashAggregateStream {
type Item = Result<RecordBatch>;
@@ -1512,7 +1488,7 @@ mod tests {
use arrow::array::{Int32Array, Int64Array, UInt32Array};
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
- use datafusion_expr::AggregateMetric;
+ use datafusion_expr::{AggregateMetric, AggregateMetrics};
use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf};
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
use datafusion_physical_expr::expressions::col;