Skip to content

Apache DataFusion SQL Query Engine

fix: Streaming aggregation produces wrong result with grouping sets (#24422)

Yongting You5c846af2caa0
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index a98c1b7bcf..57d7dba295 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -2235,7 +2235,7 @@ fn test_pushdown_grouping_sets_filter_on_common_column() {
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
output:
Ok:
- - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
"
);
@@ -2521,7 +2521,7 @@ fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input() {
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
output:
Ok:
- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])
+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]
- ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]
- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar
"
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 1a7193a7d8..97226159da 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -170,7 +170,7 @@ use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};
use crate::{
DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,
- InputOrderMode, SendableRecordBatchStream, Statistics,
+ InputOrderMode, Partitioning, SendableRecordBatchStream, Statistics,
};
use datafusion_common::config::ConfigOptions;
use parking_lot::Mutex;
@@ -1015,7 +1015,7 @@ impl AggregateExec {
.filter(|idx| group_by.groups.iter().all(|group| !group[*idx]))
.collect();
- let input_order_mode = if indices.len() == groupby_exprs.len()
+ let mut input_order_mode = if indices.len() == groupby_exprs.len()
&& !indices.is_empty()
&& group_by.groups.len() == 1
{
@@ -1026,19 +1026,29 @@ impl AggregateExec {
InputOrderMode::Linear
};
+ // To keep the ordering optimization simple, grouping sets are not supported.
+ // See [`OrderedPartialAggregateStream`] for more details about this optimization.
+ if group_by.has_grouping_set() {
+ input_order_mode = InputOrderMode::Linear;
+ }
+
// construct a map from the input expression to the output expression of the Aggregation group by
let group_expr_mapping =
ProjectionMapping::try_new(group_by.expr.clone(), &input.schema())?;
- let cache = Self::compute_properties(
- &input,
- Arc::clone(&schema),
- &group_expr_mapping,
- group_by.is_true_no_grouping(),
- &mode,
- &input_order_mode,
- aggr_expr.as_ref(),
- )?;
+ let cache = if group_by.has_grouping_set() {
+ Self::compute_grouping_set_properties(&input, Arc::clone(&schema))
+ } else {
+ Self::compute_properties(
+ &input,
+ Arc::clone(&schema),
+ &group_expr_mapping,
+ group_by.is_true_no_grouping(),
+ &mode,
+ &input_order_mode,
+ aggr_expr.as_ref(),
+ )?
+ };
let mut exec = AggregateExec {
mode,
@@ -1438,6 +1448,22 @@ impl AggregateExec {
))
}
+ fn compute_grouping_set_properties(
+ input: &Arc<dyn ExecutionPlan>,
+ schema: SchemaRef,
+ ) -> PlanProperties {
+ // Grouping-set expansion can replace group keys with nulls and adds a
+ // grouping ID, so input properties do not project through unchanged.
+ PlanProperties::new(
+ EquivalenceProperties::new(schema),
+ Partitioning::UnknownPartitioning(
+ input.output_partitioning().partition_count(),
+ ),
+ EmissionType::Final,
+ input.boundedness(),
+ )
+ }
+
pub fn input_order_mode(&self) -> &InputOrderMode {
&self.input_order_mode
}
diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.slt
index 7d893f94be..ed7b5c26f1 100644
--- a/datafusion/sqllogictest/test_files/grouping.slt
+++ b/datafusion/sqllogictest/test_files/grouping.slt
@@ -48,6 +48,97 @@ NULL A 1 0 2 1
NULL B 1 0 2 1
NULL NULL 1 1 3 3
+# Grouping sets must not use the ordered aggregation optimization even when the
+# input is ordered by a grouping expression that is present in every set.
+statement ok
+SET datafusion.execution.target_partitions = 1;
+
+statement ok
+SET datafusion.execution.enable_migration_aggregate = false;
+
+statement ok
+CREATE EXTERNAL TABLE grouping_set_ordered (
+ a0 INTEGER,
+ a INTEGER,
+ b INTEGER,
+ c INTEGER,
+ d INTEGER
+)
+STORED AS CSV
+WITH ORDER (c)
+LOCATION '../core/tests/data/window_2.csv'
+OPTIONS ('format.has_header' 'true');
+
+# An unordered AggregateExec omits `ordering_mode`; its absence below ensures
+# that grouping sets do not select ordered/streaming aggregation.
+query TT
+EXPLAIN
+SELECT b, c, grouping(b, c) AS gid
+FROM grouping_set_ordered
+GROUP BY GROUPING SETS ((b, c), (c));
+----
+logical_plan
+01)Projection: grouping_set_ordered.b, grouping_set_ordered.c, CAST(__grouping_id & UInt8(3) AS Int32) AS gid
+02)--Aggregate: groupBy=[[GROUPING SETS ((grouping_set_ordered.b, grouping_set_ordered.c), (grouping_set_ordered.c))]], aggr=[[]]
+03)----TableScan: grouping_set_ordered projection=[b, c]
+physical_plan
+01)ProjectionExec: expr=[b@0 as b, c@1 as c, CAST(__grouping_id@2 & 3 AS Int32) as gid]
+02)--AggregateExec: mode=Final, gby=[b@0 as b, c@1 as c, __grouping_id@2 as __grouping_id], aggr=[]
+03)----AggregateExec: mode=Partial, gby=[(b@0 as b, c@1 as c), (NULL as b, c@1 as c)], aggr=[]
+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c], output_ordering=[c@1 ASC NULLS LAST], file_type=csv, has_header=true
+
+# The grouping sets produce two rows for each value of c.
+query I
+SELECT c
+FROM (
+ SELECT b, c, grouping(b, c) AS gid
+ FROM grouping_set_ordered
+ GROUP BY GROUPING SETS ((b, c), (c))
+)
+ORDER BY c
+LIMIT 10;
+----
+0
+0
+1
+1
+2
+2
+3
+3
+4
+4
+
+statement ok
+SET datafusion.execution.enable_migration_aggregate = true;
+
+query I
+SELECT c
+FROM (
+ SELECT b, c, grouping(b, c) AS gid
+ FROM grouping_set_ordered
+ GROUP BY GROUPING SETS ((b, c), (c))
+)
+ORDER BY c
+LIMIT 10;
+----
+0
+0
+1
+1
+2
+2
+3
+3
+4
+4
+
+statement ok
+RESET datafusion.execution.enable_migration_aggregate;
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
# grouping_with_cube
query TTIIII
select