Apache DataFusion SQL Query Engine
fix: Streaming aggregation produces wrong result with grouping sets (#24422)
Yongting You5c846af2caa0
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex a98c1b7bcf..57d7dba295 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -2235,7 +2235,7 @@ fn test_pushdown_grouping_sets_filter_on_common_column() {- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar");@@ -2521,7 +2521,7 @@ fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input() {- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar"diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 1a7193a7d8..97226159da 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -170,7 +170,7 @@ use crate::statistics::{ChildStats, StatisticsArgs};use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};use crate::{DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,- InputOrderMode, SendableRecordBatchStream, Statistics,+ InputOrderMode, Partitioning, SendableRecordBatchStream, Statistics,};use datafusion_common::config::ConfigOptions;use parking_lot::Mutex;@@ -1015,7 +1015,7 @@ impl AggregateExec {.filter(|idx| group_by.groups.iter().all(|group| !group[*idx])).collect();- let input_order_mode = if indices.len() == groupby_exprs.len()+ let mut input_order_mode = if indices.len() == groupby_exprs.len()&& !indices.is_empty()&& group_by.groups.len() == 1{@@ -1026,19 +1026,29 @@ impl AggregateExec {InputOrderMode::Linear};+ // To keep the ordering optimization simple, grouping sets are not supported.+ // See [`OrderedPartialAggregateStream`] for more details about this optimization.+ if group_by.has_grouping_set() {+ input_order_mode = InputOrderMode::Linear;+ }+// construct a map from the input expression to the output expression of the Aggregation group bylet group_expr_mapping =ProjectionMapping::try_new(group_by.expr.clone(), &input.schema())?;- let cache = Self::compute_properties(- &input,- Arc::clone(&schema),- &group_expr_mapping,- group_by.is_true_no_grouping(),- &mode,- &input_order_mode,- aggr_expr.as_ref(),- )?;+ let cache = if group_by.has_grouping_set() {+ Self::compute_grouping_set_properties(&input, Arc::clone(&schema))+ } else {+ Self::compute_properties(+ &input,+ Arc::clone(&schema),+ &group_expr_mapping,+ group_by.is_true_no_grouping(),+ &mode,+ &input_order_mode,+ aggr_expr.as_ref(),+ )?+ };let mut exec = AggregateExec {mode,@@ -1438,6 +1448,22 @@ impl AggregateExec {))}+ fn compute_grouping_set_properties(+ input: &Arc<dyn ExecutionPlan>,+ schema: SchemaRef,+ ) -> PlanProperties {+ // Grouping-set expansion can replace group keys with nulls and adds a+ // grouping ID, so input properties do not project through unchanged.+ PlanProperties::new(+ EquivalenceProperties::new(schema),+ Partitioning::UnknownPartitioning(+ input.output_partitioning().partition_count(),+ ),+ EmissionType::Final,+ input.boundedness(),+ )+ }+pub fn input_order_mode(&self) -> &InputOrderMode {&self.input_order_mode}diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.sltindex 7d893f94be..ed7b5c26f1 100644--- a/datafusion/sqllogictest/test_files/grouping.slt+++ b/datafusion/sqllogictest/test_files/grouping.slt@@ -48,6 +48,97 @@ NULL A 1 0 2 1NULL B 1 0 2 1NULL NULL 1 1 3 3+# Grouping sets must not use the ordered aggregation optimization even when the+# input is ordered by a grouping expression that is present in every set.+statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+SET datafusion.execution.enable_migration_aggregate = false;++statement ok+CREATE EXTERNAL TABLE grouping_set_ordered (+ a0 INTEGER,+ a INTEGER,+ b INTEGER,+ c INTEGER,+ d INTEGER+)+STORED AS CSV+WITH ORDER (c)+LOCATION '../core/tests/data/window_2.csv'+OPTIONS ('format.has_header' 'true');++# An unordered AggregateExec omits `ordering_mode`; its absence below ensures+# that grouping sets do not select ordered/streaming aggregation.+query TT+EXPLAIN+SELECT b, c, grouping(b, c) AS gid+FROM grouping_set_ordered+GROUP BY GROUPING SETS ((b, c), (c));+----+logical_plan+01)Projection: grouping_set_ordered.b, grouping_set_ordered.c, CAST(__grouping_id & UInt8(3) AS Int32) AS gid+02)--Aggregate: groupBy=[[GROUPING SETS ((grouping_set_ordered.b, grouping_set_ordered.c), (grouping_set_ordered.c))]], aggr=[[]]+03)----TableScan: grouping_set_ordered projection=[b, c]+physical_plan+01)ProjectionExec: expr=[b@0 as b, c@1 as c, CAST(__grouping_id@2 & 3 AS Int32) as gid]+02)--AggregateExec: mode=Final, gby=[b@0 as b, c@1 as c, __grouping_id@2 as __grouping_id], aggr=[]+03)----AggregateExec: mode=Partial, gby=[(b@0 as b, c@1 as c), (NULL as b, c@1 as c)], aggr=[]+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c], output_ordering=[c@1 ASC NULLS LAST], file_type=csv, has_header=true++# The grouping sets produce two rows for each value of c.+query I+SELECT c+FROM (+ SELECT b, c, grouping(b, c) AS gid+ FROM grouping_set_ordered+ GROUP BY GROUPING SETS ((b, c), (c))+)+ORDER BY c+LIMIT 10;+----+0+0+1+1+2+2+3+3+4+4++statement ok+SET datafusion.execution.enable_migration_aggregate = true;++query I+SELECT c+FROM (+ SELECT b, c, grouping(b, c) AS gid+ FROM grouping_set_ordered+ GROUP BY GROUPING SETS ((b, c), (c))+)+ORDER BY c+LIMIT 10;+----+0+0+1+1+2+2+3+3+4+4++statement ok+RESET datafusion.execution.enable_migration_aggregate;++statement ok+SET datafusion.execution.target_partitions = 4;+# grouping_with_cubequery TTIIIIselect