Apache DataFusion SQL Query Engine
fix: don't declare Unique group keys for partial aggregation output (#26819)
Liang-Chi Hsieh7effb4bcc6a7
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex 05959bc62d..b6c8a6b53a 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -27,10 +27,14 @@ use datafusion_expr_common::operator::Operator;use datafusion_expr::{JoinType, SortExpr};use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_physical_expr_common::sort_expr::{- LexOrdering, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements+ LexOrdering, LexRequirement, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements};use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};+use datafusion_physical_expr::aggregate::AggregateExprBuilder;+use datafusion_functions_aggregate::sum::sum_udaf;+use datafusion_physical_plan::test::TestMemoryExec;use datafusion_physical_expr::expressions::{col, is_not_null, BinaryExpr, Column, NotExpr};+use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::repartition::RepartitionExec;@@ -46,7 +50,7 @@ use datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDownuse datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;use datafusion_physical_optimizer::projection_pushdown::ProjectionPushdown;-use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;+use datafusion_physical_optimizer::output_requirements::{OutputRequirementExec, OutputRequirements};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion::prelude::*;use arrow::array::{record_batch, Array, ArrayRef, Int32Array, RecordBatch};@@ -3684,3 +3688,66 @@ async fn test_passthrough_wrapper_projection_keeps_ordering() -> Result<()> {Ok(())}++/// A parent needs `Hash(k)` and `[k, sum]` within each partition, above a+/// Partial aggregate over input sorted on `k`. With `prefer_existing_sort`, the+/// optimizer replaces the sort with an order-preserving repartition only when+/// the repartitioned output satisfies `[k, sum]`. Partial aggregate output is+/// not unique on `k` across input partitions, so the sort must stay.+#[tokio::test]+async fn test_keep_sort_above_repartitioned_partial_aggregate() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int32, false),+ Field::new("v", DataType::Int64, false),+ ]));+ // Both partitions are sorted on `k` and contain keys 1 and 2.+ let partitions = [+ vec![record_batch!(("k", Int32, [1, 2]), ("v", Int64, [9, 5]))?],+ vec![record_batch!(("k", Int32, [1, 2]), ("v", Int64, [1, 7]))?],+ ];+ let input = TestMemoryExec::try_new(&partitions, Arc::clone(&schema), None)?+ .try_with_sort_information(vec![[sort_expr("k", &schema)].into()])?;+ let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(input)));+ let partial: Arc<dyn ExecutionPlan> = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".to_string())]),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("sum_v")+ .build()?,+ )],+ vec![None],+ input,+ Arc::clone(&schema),+ )?);++ let k: Arc<dyn PhysicalExpr> = Arc::new(Column::new("k", 0));+ let sum: Arc<dyn PhysicalExpr> =+ Arc::new(Column::new(partial.schema().field(1).name(), 1));+ let requirement = LexRequirement::new([+ PhysicalSortRequirement::new(Arc::clone(&k), Some(SortOptions::default())),+ PhysicalSortRequirement::new(sum, Some(SortOptions::default())),+ ])+ .unwrap();+ let plan: Arc<dyn ExecutionPlan> = Arc::new(OutputRequirementExec::new(+ partial,+ Some(OrderingRequirements::new(requirement)),+ Distribution::KeyPartitioned(vec![k]),+ None,+ ));++ let mut config = ConfigOptions::new();+ config.execution.target_partitions = 2;+ config.optimizer.prefer_existing_sort = true;+ let optimized = EnsureRequirements::new().optimize(plan, &config)?;+ let optimized = OutputRequirements::new_remove_mode().optimize(optimized, &config)?;+ assert_snapshot!(displayable(optimized.as_ref()).indent(true), @r"+ SortExec: expr=[k@0 ASC, sum_v[sum]@1 ASC], preserve_partitioning=[true]+ RepartitionExec: partitioning=Hash([k@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[k@0 as k], aggr=[sum_v], group_clustering_mode=Full+ DataSourceExec: partitions=2, partition_sizes=[1, 1], output_ordering=k@0 ASC+ ");++ Ok(())+}diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rsindex ad3c551a93..4d3dc89f7a 100644--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs@@ -130,7 +130,10 @@ pub struct EquivalenceProperties {/// constants/duplicates and in standard form) and a map associating leading/// terms with full sort expressions.oeq_cache: OrderingEquivalenceCache,- /// Table constraints that factor in equivalence calculations.+ /// Table constraints that factor in equivalence calculations. Unlike+ /// constants, constraints have no per-partition form: each one must hold+ /// across all output partitions, since operators that merge partitions+ /// keep them.constraints: Constraints,/// Schema associated with this object.schema: SchemaRef,diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex a5497765c9..d3c85deb22 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1466,22 +1466,25 @@ impl AggregateExec {eq_properties.add_constants(new_constants)?;}- // Group by expression will be a distinct value after the aggregation.- // Add it into the constraint set.- let mut constraints = eq_properties.constraints().to_vec();- let new_constraint = Constraint::Unique(- group_expr_mapping- .iter()- .flat_map(|(_, target_cols)| {- target_cols.iter().filter_map(|(expr, _)| {- expr.downcast_ref::<Column>().map(|c| c.index())+ // Only final output is unique on the group keys. Partial output can repeat a+ // group within a partition (early emission, skipped partial aggregation) and+ // across partitions once they are merged.+ if mode.output_mode() == AggregateOutputMode::Final {+ let mut constraints = eq_properties.constraints().to_vec();+ let new_constraint = Constraint::Unique(+ group_expr_mapping+ .iter()+ .flat_map(|(_, target_cols)| {+ target_cols.iter().filter_map(|(expr, _)| {+ expr.downcast_ref::<Column>().map(|c| c.index())+ })})- })- .collect(),- );- constraints.push(new_constraint);- eq_properties =- eq_properties.with_constraints(Constraints::new_unverified(constraints));+ .collect(),+ );+ constraints.push(new_constraint);+ eq_properties =+ eq_properties.with_constraints(Constraints::new_unverified(constraints));+ }// Get output partitioning:let input_partitioning = input.output_partitioning().clone();@@ -5391,6 +5394,19 @@ mod tests {aggregate.required_input_ordering(),vec![Some(OrderingRequirements::new_soft(input_ordering.into()))]);+ let expected_constraints = match mode.output_mode() {+ AggregateOutputMode::Final => {+ Constraints::new_unverified(vec![Constraint::Unique(vec![0, 1])])+ }+ AggregateOutputMode::Partial => Constraints::default(),+ };+ assert_eq!(+ aggregate+ .properties()+ .equivalence_properties()+ .constraints(),+ &expected_constraints+ );Ok(())}