Skip to content

Apache DataFusion SQL Query Engine

fix: look up a node's outputs by column, not by flat name, in PushDownFilter (#26783)

RAGHVENDRA KUMAR YADAV6341b5c9fc30
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index 74ae184b5e..ab3ee7a68b 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -1116,9 +1116,15 @@ impl OptimizerRule for PushDownFilter {
// As for plan Filter: Column(a+b) > 0 -- Agg: groupby:[Column(a)+Column(b)]
// After push, we need to replace `a+b` with Column(a)+Column(b)
// So we need create a replace_map, add {`a+b` --> Expr(Column(a)+Column(b))}
+ //
+ // Keyed by the output column, the same key `group_expr_columns`
+ // is built from, so a predicate admitted below cannot miss its
+ // replacement. See `replace_cols_by_column` for why the flat
+ // name is not enough.
let mut replace_map = HashMap::with_capacity(agg.group_expr.len());
for expr in &agg.group_expr {
- replace_map.insert(expr.schema_name().to_string(), unalias(expr));
+ let (relation, name) = expr.qualified_name();
+ replace_map.insert(Column::new(relation, name), unalias(expr));
}
let predicates = split_conjunction_owned(filter.predicate);
@@ -1127,7 +1133,7 @@ impl OptimizerRule for PushDownFilter {
for expr in predicates {
let cols = expr.column_refs();
if cols.iter().all(|c| group_expr_columns.contains(c)) {
- push_predicates.push(replace_cols_by_name(expr, &replace_map)?);
+ push_predicates.push(replace_cols_by_column(expr, &replace_map)?);
} else {
keep_predicates.push(expr);
}
@@ -1544,12 +1550,17 @@ fn rewrite_projection(
// into a filter causes optimizer instability — ExtractLeafExpressions will
// undo the push-down, creating an infinite loop that runs until the
// iteration limit is hit).
- let (non_pushable_map, pushable_map) = projection
+ //
+ // Keyed by the output column rather than its flat name, for the reason
+ // given on `replace_cols_by_column`. `EliminateGroupByConstant` can leave a
+ // projection that outputs both `t.a` and `CAST(t.a AS ...)` under that one
+ // name.
+ let (non_pushable_map, pushable_map): (HashMap<_, _>, HashMap<_, _>) = projection
.schema
.iter()
.zip(projection.expr.iter())
.map(|((qualifier, field), expr)| {
- (qualified_name(qualifier, field.name()), unalias(expr))
+ (Column::new(qualifier.cloned(), field.name()), unalias(expr))
})
.partition(|(_, value)| {
value.is_volatile()
@@ -1559,7 +1570,7 @@ fn rewrite_projection(
let mut push_predicates = vec![];
let mut keep_predicates = vec![];
for expr in predicates {
- if contain(&expr, &non_pushable_map) {
+ if contains_column(&expr, &non_pushable_map) {
keep_predicates.push(expr);
} else {
push_predicates.push(expr);
@@ -1570,7 +1581,7 @@ fn rewrite_projection(
// re-write all filters based on this projection
// E.g. in `Filter: b\n Projection: a > 1 as b`, we can swap them, but the filter must be "a > 1"
projection.input = Arc::new(LogicalPlan::Filter(Filter::new(
- replace_cols_by_name(expr, &pushable_map)?,
+ replace_cols_by_column(expr, &pushable_map)?,
projection.input,
)));
@@ -1630,6 +1641,28 @@ pub fn replace_cols_by_name(
.data()
}
+/// Replaces column references by the expression the node below computes for
+/// that column.
+///
+/// Keyed by [`Column`] rather than by its flat name, because a qualified column
+/// `t.a` and an unqualified field named `t.a` are different columns that share a
+/// flat name, and a node can output both.
+fn replace_cols_by_column(
+ e: Expr,
+ replace_map: &HashMap<Column, impl AsRef<Expr>>,
+) -> Result<Expr> {
+ e.transform_up(|expr| {
+ if let Expr::Column(c) = &expr
+ && let Some(new_expr) = replace_map.get(c)
+ {
+ Ok(Transformed::yes(new_expr.as_ref().clone()))
+ } else {
+ Ok(Transformed::no(expr))
+ }
+ })
+ .data()
+}
+
/// Unalias expression reference.
fn unalias(expr: &Expr) -> &Expr {
if let Expr::Alias(alias) = expr {
@@ -1640,11 +1673,9 @@ fn unalias(expr: &Expr) -> &Expr {
}
/// check whether the expression uses the columns in `check_map`.
-fn contain<T>(e: &Expr, check_map: &HashMap<String, T>) -> bool {
- e.exists(|expr| {
- Ok(matches!(expr, Expr::Column(c) if check_map.contains_key(&c.flat_name())))
- })
- .unwrap()
+fn contains_column<T>(e: &Expr, check_map: &HashMap<Column, T>) -> bool {
+ e.exists(|expr| Ok(matches!(expr, Expr::Column(c) if check_map.contains_key(c))))
+ .unwrap()
}
fn with_filters(predicates: Vec<Expr>, plan: LogicalPlan) -> LogicalPlan {
diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt
index f1fa009e2c..17fad4ba55 100644
--- a/datafusion/sqllogictest/test_files/group_by.slt
+++ b/datafusion/sqllogictest/test_files/group_by.slt
@@ -5956,3 +5956,97 @@ y! 2 3 1
statement ok
DROP TABLE group_by_struct_requalify;
+
+# `GROUP BY a, CAST(a AS VARCHAR)` makes the aggregate output two columns that
+# share a flat name, and PushDownFilter used to rewrite a predicate on `a` to
+# whichever of them came last, which gave `Utf8View > Int32` for a valid query.
+# https://github.com/apache/datafusion/issues/26619
+statement ok
+CREATE TABLE group_by_duplicate_name (a INT) AS VALUES (1), (2);
+
+# the aggregate outputs both names
+query I
+SELECT a FROM group_by_duplicate_name
+GROUP BY a, CAST(a AS VARCHAR)
+HAVING a > (SELECT min(a) FROM group_by_duplicate_name);
+----
+2
+
+query I
+SELECT a FROM group_by_duplicate_name GROUP BY a, CAST(a AS VARCHAR) HAVING a > 1;
+----
+2
+
+# the cast is kept in the output, which changes what the later passes see
+query IT
+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name
+GROUP BY a, CAST(a AS VARCHAR)
+HAVING a > (SELECT min(a) FROM group_by_duplicate_name)
+ORDER BY a;
+----
+2 2
+
+# A control: with the cast first, the plain column is the one the old lookup
+# kept, so this ordering passed before the change. It pins the fact that the
+# answer no longer depends on the order of the group expressions.
+query IT
+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name
+GROUP BY CAST(a AS VARCHAR), a
+HAVING a > 1
+ORDER BY a;
+----
+2 2
+
+# A control: a predicate on the cast resolved to it either way, so this also
+# passed before the change. It pins the other side of the lookup.
+query IT
+SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name
+GROUP BY a, CAST(a AS VARCHAR)
+HAVING CAST(a AS VARCHAR) > '1'
+ORDER BY a;
+----
+2 2
+
+# The HAVING cases above are all decided in the Aggregate branch. These two
+# reach the projection that EliminateGroupByConstant leaves behind, which
+# outputs `a` and `CAST(a AS VARCHAR)` under one flat name. The predicate has to
+# survive the first pass for the projection to exist by the time it is pushed:
+# once because an OR with an aggregate blocks it, once behind a LIMIT.
+query ITI
+SELECT a, CAST(a AS VARCHAR) AS s, count(*) AS n FROM group_by_duplicate_name
+GROUP BY a, CAST(a AS VARCHAR)
+HAVING a > 1 OR count(*) > 1;
+----
+2 2 1
+
+query IT
+SELECT * FROM (
+ SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name
+ GROUP BY a, CAST(a AS VARCHAR)
+ LIMIT 10
+) WHERE a > 1;
+----
+2 2
+
+# the predicate still reaches the scan
+query TT
+EXPLAIN SELECT a, CAST(a AS VARCHAR) AS s FROM group_by_duplicate_name
+GROUP BY a, CAST(a AS VARCHAR)
+HAVING a > 1;
+----
+logical_plan
+01)Projection: group_by_duplicate_name.a, CAST(group_by_duplicate_name.a AS Utf8View) AS s
+02)--Aggregate: groupBy=[[group_by_duplicate_name.a]], aggr=[[]]
+03)----Filter: group_by_duplicate_name.a > Int32(1)
+04)------TableScan: group_by_duplicate_name projection=[a]
+physical_plan
+01)ProjectionExec: expr=[a@0 as a, CAST(a@0 AS Utf8View) as s]
+02)--AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+03)----RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4
+04)------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+06)----------FilterExec: a@0 > 1
+07)------------DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+DROP TABLE group_by_duplicate_name;