Skip to content

Apache DataFusion SQL Query Engine

fix: don't declare Unique group keys for partial aggregation output (#26819)

Liang-Chi Hsieh7effb4bcc6a7
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index 05959bc62d..b6c8a6b53a 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -27,10 +27,14 @@ use datafusion_expr_common::operator::Operator;
use datafusion_expr::{JoinType, SortExpr};
use datafusion_execution::object_store::ObjectStoreUrl;
use datafusion_physical_expr_common::sort_expr::{
- LexOrdering, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements
+ LexOrdering, LexRequirement, PhysicalSortExpr, PhysicalSortRequirement, OrderingRequirements
};
use datafusion_physical_expr::{Distribution, Partitioning, PhysicalExpr};
+use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+use datafusion_functions_aggregate::sum::sum_udaf;
+use datafusion_physical_plan::test::TestMemoryExec;
use datafusion_physical_expr::expressions::{col, is_not_null, BinaryExpr, Column, NotExpr};
+use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};
use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::repartition::RepartitionExec;
@@ -46,7 +50,7 @@ use datafusion_physical_optimizer::enforce_sorting::sort_pushdown::{SortPushDown
use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
use datafusion_physical_optimizer::limit_pushdown::LimitPushdown;
use datafusion_physical_optimizer::projection_pushdown::ProjectionPushdown;
-use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;
+use datafusion_physical_optimizer::output_requirements::{OutputRequirementExec, OutputRequirements};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion::prelude::*;
use arrow::array::{record_batch, Array, ArrayRef, Int32Array, RecordBatch};
@@ -3684,3 +3688,66 @@ async fn test_passthrough_wrapper_projection_keeps_ordering() -> Result<()> {
Ok(())
}
+
+/// A parent needs `Hash(k)` and `[k, sum]` within each partition, above a
+/// Partial aggregate over input sorted on `k`. With `prefer_existing_sort`, the
+/// optimizer replaces the sort with an order-preserving repartition only when
+/// the repartitioned output satisfies `[k, sum]`. Partial aggregate output is
+/// not unique on `k` across input partitions, so the sort must stay.
+#[tokio::test]
+async fn test_keep_sort_above_repartitioned_partial_aggregate() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("k", DataType::Int32, false),
+ Field::new("v", DataType::Int64, false),
+ ]));
+ // Both partitions are sorted on `k` and contain keys 1 and 2.
+ let partitions = [
+ vec![record_batch!(("k", Int32, [1, 2]), ("v", Int64, [9, 5]))?],
+ vec![record_batch!(("k", Int32, [1, 2]), ("v", Int64, [1, 7]))?],
+ ];
+ let input = TestMemoryExec::try_new(&partitions, Arc::clone(&schema), None)?
+ .try_with_sort_information(vec![[sort_expr("k", &schema)].into()])?;
+ let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(input)));
+ let partial: Arc<dyn ExecutionPlan> = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".to_string())]),
+ vec![Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("sum_v")
+ .build()?,
+ )],
+ vec![None],
+ input,
+ Arc::clone(&schema),
+ )?);
+
+ let k: Arc<dyn PhysicalExpr> = Arc::new(Column::new("k", 0));
+ let sum: Arc<dyn PhysicalExpr> =
+ Arc::new(Column::new(partial.schema().field(1).name(), 1));
+ let requirement = LexRequirement::new([
+ PhysicalSortRequirement::new(Arc::clone(&k), Some(SortOptions::default())),
+ PhysicalSortRequirement::new(sum, Some(SortOptions::default())),
+ ])
+ .unwrap();
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(OutputRequirementExec::new(
+ partial,
+ Some(OrderingRequirements::new(requirement)),
+ Distribution::KeyPartitioned(vec![k]),
+ None,
+ ));
+
+ let mut config = ConfigOptions::new();
+ config.execution.target_partitions = 2;
+ config.optimizer.prefer_existing_sort = true;
+ let optimized = EnsureRequirements::new().optimize(plan, &config)?;
+ let optimized = OutputRequirements::new_remove_mode().optimize(optimized, &config)?;
+ assert_snapshot!(displayable(optimized.as_ref()).indent(true), @r"
+ SortExec: expr=[k@0 ASC, sum_v[sum]@1 ASC], preserve_partitioning=[true]
+ RepartitionExec: partitioning=Hash([k@0], 2), input_partitions=2
+ AggregateExec: mode=Partial, gby=[k@0 as k], aggr=[sum_v], group_clustering_mode=Full
+ DataSourceExec: partitions=2, partition_sizes=[1, 1], output_ordering=k@0 ASC
+ ");
+
+ Ok(())
+}
diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs
index ad3c551a93..4d3dc89f7a 100644
--- a/datafusion/physical-expr/src/equivalence/properties/mod.rs
+++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs
@@ -130,7 +130,10 @@ pub struct EquivalenceProperties {
/// constants/duplicates and in standard form) and a map associating leading
/// terms with full sort expressions.
oeq_cache: OrderingEquivalenceCache,
- /// Table constraints that factor in equivalence calculations.
+ /// Table constraints that factor in equivalence calculations. Unlike
+ /// constants, constraints have no per-partition form: each one must hold
+ /// across all output partitions, since operators that merge partitions
+ /// keep them.
constraints: Constraints,
/// Schema associated with this object.
schema: SchemaRef,
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index a5497765c9..d3c85deb22 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -1466,22 +1466,25 @@ impl AggregateExec {
eq_properties.add_constants(new_constants)?;
}
- // Group by expression will be a distinct value after the aggregation.
- // Add it into the constraint set.
- let mut constraints = eq_properties.constraints().to_vec();
- let new_constraint = Constraint::Unique(
- group_expr_mapping
- .iter()
- .flat_map(|(_, target_cols)| {
- target_cols.iter().filter_map(|(expr, _)| {
- expr.downcast_ref::<Column>().map(|c| c.index())
+ // Only final output is unique on the group keys. Partial output can repeat a
+ // group within a partition (early emission, skipped partial aggregation) and
+ // across partitions once they are merged.
+ if mode.output_mode() == AggregateOutputMode::Final {
+ let mut constraints = eq_properties.constraints().to_vec();
+ let new_constraint = Constraint::Unique(
+ group_expr_mapping
+ .iter()
+ .flat_map(|(_, target_cols)| {
+ target_cols.iter().filter_map(|(expr, _)| {
+ expr.downcast_ref::<Column>().map(|c| c.index())
+ })
})
- })
- .collect(),
- );
- constraints.push(new_constraint);
- eq_properties =
- eq_properties.with_constraints(Constraints::new_unverified(constraints));
+ .collect(),
+ );
+ constraints.push(new_constraint);
+ eq_properties =
+ eq_properties.with_constraints(Constraints::new_unverified(constraints));
+ }
// Get output partitioning:
let input_partitioning = input.output_partitioning().clone();
@@ -5391,6 +5394,19 @@ mod tests {
aggregate.required_input_ordering(),
vec![Some(OrderingRequirements::new_soft(input_ordering.into()))]
);
+ let expected_constraints = match mode.output_mode() {
+ AggregateOutputMode::Final => {
+ Constraints::new_unverified(vec![Constraint::Unique(vec![0, 1])])
+ }
+ AggregateOutputMode::Partial => Constraints::default(),
+ };
+ assert_eq!(
+ aggregate
+ .properties()
+ .equivalence_properties()
+ .constraints(),
+ &expected_constraints
+ );
Ok(())
}