Skip to content

Apache DataFusion SQL Query Engine

fix: keep EliminateGroupByConstant from requalifying group expressions (#26356)

Adrian Garcia Badaracco4f2dbf70be7e
diff --git a/datafusion/optimizer/src/eliminate_group_by_constant.rs b/datafusion/optimizer/src/eliminate_group_by_constant.rs
index a5f8d2ca47..a2cdc2f70f 100644
--- a/datafusion/optimizer/src/eliminate_group_by_constant.rs
+++ b/datafusion/optimizer/src/eliminate_group_by_constant.rs
@@ -21,10 +21,11 @@ use crate::optimizer::ApplyOrder;
use crate::{OptimizerConfig, OptimizerRule};
use std::collections::HashSet;
+use std::sync::Arc;
use datafusion_common::Result;
use datafusion_common::tree_node::Transformed;
-use datafusion_expr::{Aggregate, Expr, LogicalPlan, LogicalPlanBuilder, Volatility};
+use datafusion_expr::{Aggregate, Expr, LogicalPlan, Projection, Volatility};
/// Optimizer rule that removes constant expressions from `GROUP BY` clause
/// and places additional projection on top of aggregation, to preserve
@@ -56,10 +57,12 @@ impl OptimizerRule for EliminateGroupByConstant {
})
.collect();
- let (redundant, required): (Vec<_>, Vec<_>) = aggregate
+ let is_redundant: Vec<bool> = aggregate
.group_expr
.iter()
- .partition(|expr| is_redundant_group_expr(expr, &group_by_columns));
+ .map(|expr| is_redundant_group_expr(expr, &group_by_columns))
+ .collect();
+ let redundant_count = is_redundant.iter().filter(|r| **r).count();
// Return now if no simplification can be done. We also bail out
// if applying the optimization would eliminate all of the
// grouping expressions (e.g., GROUP BY on only constant
@@ -67,22 +70,65 @@ impl OptimizerRule for EliminateGroupByConstant {
// ungrouped aggregate, which changes query semantics (grouped
// aggregates produce an empty result set on an empty input,
// whereas ungrouped aggregates return a single row).
- if redundant.is_empty() || required.is_empty() {
+ if redundant_count == 0 || redundant_count == aggregate.group_expr.len() {
return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate)));
}
- let simplified_aggregate = LogicalPlan::Aggregate(Aggregate::try_new(
- aggregate.input,
- required.into_iter().cloned().collect(),
- aggregate.aggr_expr.clone(),
- )?);
-
- let projection_expr =
- aggregate.group_expr.into_iter().chain(aggregate.aggr_expr);
+ let original_schema = Arc::clone(&aggregate.schema);
+ let required = aggregate
+ .group_expr
+ .iter()
+ .zip(&is_redundant)
+ .filter(|(_, redundant)| !**redundant)
+ .map(|(expr, _)| expr.clone())
+ .collect();
+ let simplified_aggregate =
+ Aggregate::try_new(aggregate.input, required, aggregate.aggr_expr)?;
+ let simplified_schema = Arc::clone(&simplified_aggregate.schema);
+
+ // Refer to the output columns of the simplified aggregate by
+ // position. Do not re-plan the original group expressions on top
+ // of it: an expression can refer to an input column through a
+ // name that the aggregate output does not carry, so a rebuilt
+ // projection does not always resolve. A redundant expression
+ // refers only to bare GROUP BY columns, which the simplified
+ // aggregate outputs unchanged.
+ let mut projection_expr =
+ Vec::with_capacity(original_schema.fields().len());
+ let mut next_output = 0;
+ for (i, (expr, redundant)) in aggregate
+ .group_expr
+ .into_iter()
+ .zip(is_redundant)
+ .enumerate()
+ {
+ let (qualifier, field) = original_schema.qualified_field(i);
+ let expr = if redundant {
+ expr
+ } else {
+ let column =
+ Expr::from(simplified_schema.qualified_field(next_output));
+ next_output += 1;
+ column
+ };
+ let expr = if expr.qualified_name()
+ == (qualifier.cloned(), field.name().clone())
+ {
+ expr
+ } else {
+ expr.alias_qualified(qualifier.cloned(), field.name())
+ };
+ projection_expr.push(expr);
+ }
+ for i in next_output..simplified_schema.fields().len() {
+ projection_expr
+ .push(Expr::from(simplified_schema.qualified_field(i)));
+ }
- let projection = LogicalPlanBuilder::from(simplified_aggregate)
- .project(projection_expr)?
- .build()?;
+ let projection = LogicalPlan::Projection(Projection::try_new(
+ projection_expr,
+ Arc::new(LogicalPlan::Aggregate(simplified_aggregate)),
+ )?);
Ok(Transformed::yes(projection))
}
@@ -220,6 +266,33 @@ mod tests {
")
}
+ /// A required GROUP BY expression refers to an input column without the
+ /// qualifier that the input schema gives it. `ExtractLeafExpressions`
+ /// creates such references when it moves an extraction projection below a
+ /// `SubqueryAlias`. The rule must keep the plan valid.
+ #[test]
+ fn test_eliminate_with_unqualified_reference_to_qualified_input() -> Result<()> {
+ let input = LogicalPlanBuilder::from(test_table_scan()?)
+ .alias("m")?
+ .build()?;
+ let plan = LogicalPlan::Aggregate(Aggregate::try_new(
+ Arc::new(input),
+ vec![
+ Expr::Column(datafusion_common::Column::new_unqualified("b")).alias("k"),
+ col("m.a"),
+ col("m.a") + lit(1u32),
+ ],
+ vec![count(col("m.c"))],
+ )?);
+
+ assert_optimized_plan_equal!(plan, @r"
+ Projection: k, m.a, m.a + UInt32(1), count(m.c)
+ Aggregate: groupBy=[[b AS k, m.a]], aggr=[[count(m.c)]]
+ SubqueryAlias: m
+ TableScan: test
+ ")
+ }
+
#[test]
fn test_no_op_only_constant_with_aggregate() -> Result<()> {
let scan = test_table_scan()?;
diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt
index 281038f62e..64c6b04d41 100644
--- a/datafusion/sqllogictest/test_files/group_by.slt
+++ b/datafusion/sqllogictest/test_files/group_by.slt
@@ -5922,3 +5922,22 @@ from narrow_decimal_multi_group group by 1, 2 order by 1, 2;
statement ok
drop table narrow_decimal_multi_group;
+
+# EliminateGroupByConstant must not requalify group expressions over a
+# struct field accessed through a subquery alias
+# https://github.com/apache/datafusion/pull/26356
+statement ok
+CREATE TABLE group_by_struct_requalify AS
+SELECT * FROM (VALUES (named_struct('a', 'x', 'b', 1)), (named_struct('a', 'y', 'b', 2))) AS v(s);
+
+query TIII
+SELECT m.s['a'] || '!' AS k1, m.s['b'] AS k2, m.s['b'] + 1 AS k3, count(*) AS n
+FROM (SELECT * FROM group_by_struct_requalify) m
+GROUP BY m.s['a'] || '!', m.s['b'], m.s['b'] + 1
+ORDER BY k2;
+----
+x! 1 2 1
+y! 2 3 1
+
+statement ok
+DROP TABLE group_by_struct_requalify;