Apache DataFusion SQL Query Engine
fix: look up a node's outputs by column, not by flat name, in PushDownFilter (#26783)
RAGHVENDRA KUMAR YADAV6341b5c9fc30
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex 74ae184b5e..ab3ee7a68b 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -1116,9 +1116,15 @@ impl OptimizerRule for PushDownFilter {// As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]// After push, we need to replace `a+b` with Column(a)+Column(b)// So we need create a replace_map, add {`a+b` --> Expr(Column(a)+Column(b))}+ //+ // Keyed by the output column, the same key `group_expr_columns`+ // is built from, so a predicate admitted below cannot miss its+ // replacement. See `replace_cols_by_column` for why the flat+ // name is not enough.let mut replace_map = HashMap::with_capacity(agg.group_expr.len());for expr in &agg.group_expr {- replace_map.insert(expr.schema_name().to_string(), unalias(expr));+ let (relation, name) = expr.qualified_name();+ replace_map.insert(Column::new(relation, name), unalias(expr));}let predicates = split_conjunction_owned(filter.predicate);@@ -1127,7 +1133,7 @@ impl OptimizerRule for PushDownFilter {for expr in predicates {let cols = expr.column_refs();if cols.iter().all(|c| group_expr_columns.contains(c)) {- push_predicates.push(replace_cols_by_name(expr, &replace_map)?);+ push_predicates.push(replace_cols_by_column(expr, &replace_map)?);} else {keep_predicates.push(expr);}@@ -1544,12 +1550,17 @@ fn rewrite_projection(// into a filter causes optimizer instability — ExtractLeafExpressions will// undo the push-down, creating an infinite loop that runs until the// iteration limit is hit).- let (non_pushable_map, pushable_map) = projection+ //+ // Keyed by the output column rather than its flat name, for the reason+ // given on `replace_cols_by_column`. `EliminateGroupByConstant` can leave a+ // projection that outputs both `t.a` and `CAST(t.a AS ...)` under that one+ // name.+ let (non_pushable_map, pushable_map): (HashMap<_, _>, HashMap<_, _>) = projection.schema.iter().zip(projection.expr.iter()).map(|((qualifier, field), expr)| {- (qualified_name(qualifier, field.name()), unalias(expr))+ (Column::new(qualifier.cloned(), field.name()), unalias(expr))}).partition(|(_, value)| {value.is_volatile()@@ -1559,7 +1570,7 @@ fn rewrite_projection(let mut push_predicates = vec![];let mut keep_predicates = vec![];for expr in predicates {- if contain(&expr, &non_pushable_map) {+ if contains_column(&expr, &non_pushable_map) {keep_predicates.push(expr);} else {push_predicates.push(expr);@@ -1570,7 +1581,7 @@ fn rewrite_projection(// re-write all filters based on this projection// E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"projection.input = Arc::new(LogicalPlan::Filter(Filter::new(- replace_cols_by_name(expr, &pushable_map)?,+ replace_cols_by_column(expr, &pushable_map)?,projection.input,)));@@ -1630,6 +1641,28 @@ pub fn replace_cols_by_name(.data()}+/// Replaces column references by the expression the node below computes for+/// that column.+///+/// Keyed by [`Column`] rather than by its flat name, because a qualified column+/// `t.a` and an unqualified field named `t.a` are different columns that share a+/// flat name, and a node can output both.+fn replace_cols_by_column(+ e: Expr,+ replace_map: &HashMap<Column, impl AsRef<Expr>>,+) -> Result<Expr> {+ e.transform_up(|expr| {+ if let Expr::Column(c) = &expr+ && let Some(new_expr) = replace_map.get(c)+ {+ Ok(Transformed::yes(new_expr.as_ref().clone()))+ } else {+ Ok(Transformed::no(expr))+ }+ })+ .data()+}+/// Unalias expression reference.fn unalias(expr: &Expr) -> &Expr {if let Expr::Alias(alias) = expr {@@ -1640,11 +1673,9 @@ fn unalias(expr: &Expr) -> &Expr {}/// check whether the expression uses the columns in `check_map`.-fn contain<T>(e: &Expr, check_map: &HashMap<String, T>) -> bool {- e.exists(|expr| {- Ok(matches!(expr, Expr::Column(c) if check_map.contains_key(&c.flat_name())))- })- .unwrap()+fn contains_column<T>(e: &Expr, check_map: &HashMap<Column, T>) -> bool {+ e.exists(|expr| Ok(matches!(expr, Expr::Column(c) if check_map.contains_key(c))))+ .unwrap()}fn with_filters(predicates: Vec<Expr>, plan: LogicalPlan) -> LogicalPlan {diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex f1fa009e2c..17fad4ba55 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -5956,3 +5956,97 @@ y! 2 3 1statement okDROP TABLE group_by_struct_requalify;++# `GROUP BY a, CAST(a AS VARCHAR)` makes the aggregate output two columns that+# share a flat name, and PushDownFilter used to rewrite a predicate on `a` to+# whichever of them came last, which gave `Utf8View > Int32` for a valid query.+# https://github.com/apache/datafusion/issues/26619+statement ok+CREATE TABLE group_by_duplicate_name (a INT) AS VALUES (1), (2);++# the aggregate outputs both names+query I+SELECT a FROM group_by_duplicate_name+GROUP BY a, CAST(a AS VARCHAR)+HAVING a > (SELECT min(a) FROM group_by_duplicate_name);+----+2++query I+SELECT a FROM group_by_duplicate_name GROUP BY a, CAST(a AS VARCHAR) HAVING a > 1;+----+2++# the cast is kept in the output, which changes what the later passes see+query IT+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name+GROUP BY a, CAST(a AS VARCHAR)+HAVING a > (SELECT min(a) FROM group_by_duplicate_name)+ORDER BY a;+----+2 2++# A control: with the cast first, the plain column is the one the old lookup+# kept, so this ordering passed before the change. It pins the fact that the+# answer no longer depends on the order of the group expressions.+query IT+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name+GROUP BY CAST(a AS VARCHAR), a+HAVING a > 1+ORDER BY a;+----+2 2++# A control: a predicate on the cast resolved to it either way, so this also+# passed before the change. It pins the other side of the lookup.+query IT+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name+GROUP BY a, CAST(a AS VARCHAR)+HAVING CAST(a AS VARCHAR) > '1'+ORDER BY a;+----+2 2++# The HAVING cases above are all decided in the Aggregate branch. These two+# reach the projection that EliminateGroupByConstant leaves behind, which+# outputs `a` and `CAST(a AS VARCHAR)` under one flat name. The predicate has to+# survive the first pass for the projection to exist by the time it is pushed:+# once because an OR with an aggregate blocks it, once behind a LIMIT.+query ITI+SELECT a, CAST(a AS VARCHAR) AS s, count(*) AS n FROM group_by_duplicate_name+GROUP BY a, CAST(a AS VARCHAR)+HAVING a > 1 OR count(*) > 1;+----+2 2 1++query IT+SELECT * FROM (+ SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name+ GROUP BY a, CAST(a AS VARCHAR)+ LIMIT 10+) WHERE a > 1;+----+2 2++# the predicate still reaches the scan+query TT+EXPLAIN SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name+GROUP BY a, CAST(a AS VARCHAR)+HAVING a > 1;+----+logical_plan+01)Projection: group_by_duplicate_name.a, CAST(group_by_duplicate_name.a AS Utf8View) AS s+02)--Aggregate: groupBy=[[group_by_duplicate_name.a]], aggr=[[]]+03)----Filter: group_by_duplicate_name.a > Int32(1)+04)------TableScan: group_by_duplicate_name projection=[a]+physical_plan+01)ProjectionExec: expr=[a@0 as a, CAST(a@0 AS Utf8View) as s]+02)--AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+03)----RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4+04)------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+06)----------FilterExec: a@0 > 1+07)------------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+DROP TABLE group_by_duplicate_name;