Apache DataFusion SQL Query Engine
try to remove redundant alias in expression rewriter and select (#20867)
Burak Şen385d9dbe7106
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex b1ee8b09b9..80bbde1f6b 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -3004,7 +3004,7 @@ async fn test_count_wildcard_on_sort() -> Result<()> {+---------------+------------------------------------------------------------------------------------+| plan_type | plan |+---------------+------------------------------------------------------------------------------------+- | logical_plan | Sort: count(*) AS count(*) ASC NULLS LAST |+ | logical_plan | Sort: count(*) ASC NULLS LAST || | Projection: t1.b, count(Int64(1)) AS count(*) || | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1))]] || | TableScan: t1 projection=[b] |diff --git a/datafusion/expr/src/expr_rewriter/order_by.rs b/datafusion/expr/src/expr_rewriter/order_by.rsindex a897e56d27..720788113c 100644--- a/datafusion/expr/src/expr_rewriter/order_by.rs+++ b/datafusion/expr/src/expr_rewriter/order_by.rs@@ -21,9 +21,7 @@ use crate::expr::Alias;use crate::expr_rewriter::normalize_col;use crate::{Cast, Expr, LogicalPlan, TryCast, expr::Sort};-use datafusion_common::tree_node::{- Transformed, TransformedResult, TreeNode, TreeNodeRecursion,-};+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{Column, Result};/// Rewrite sort on aggregate expressions to sort on the column of aggregate output@@ -104,29 +102,27 @@ fn rewrite_in_terms_of_projection(let search_col = Expr::Column(Column::new_unqualified(name));- // look for the column named the same as this expr- let mut found = None;- for proj_expr in proj_exprs {- proj_expr.apply(|e| {- if expr_match(&search_col, e) {- found = Some(e.clone());- return Ok(TreeNodeRecursion::Stop);- }- Ok(TreeNodeRecursion::Continue)- })?;- }+ // Search only top-level projection expressions for a match.+ // We intentionally avoid a recursive search (e.g. `apply`) to+ // prevent matching sub-expressions of composites like+ // `min(c2) + max(c3)` when the ORDER BY is just `min(c2)`.+ let found = proj_exprs+ .iter()+ .find(|proj_expr| expr_match(&search_col, proj_expr));if let Some(found) = found {+ let (qualifier, field_name) = found.qualified_name();+ let col = Expr::Column(Column::new(qualifier, field_name));return Ok(Transformed::yes(match normalized_expr {Expr::Cast(Cast { expr: _, field }) => Expr::Cast(Cast {- expr: Box::new(found),+ expr: Box::new(col),field,}),Expr::TryCast(TryCast { expr: _, field }) => Expr::TryCast(TryCast {- expr: Box::new(found),+ expr: Box::new(col),field,}),- _ => found,+ _ => col,}));}@@ -160,7 +156,10 @@ mod test {use super::*;use crate::test::function_stub::avg;+ use crate::test::function_stub::count;+ use crate::test::function_stub::max;use crate::test::function_stub::min;+ use crate::test::function_stub::sum;#[test]fn rewrite_sort_cols_by_agg() {@@ -242,17 +241,14 @@ mod test {TestCase {desc: r#"c1 + min(c2) --> "c1 + min(c2)" -- (column *named* "min(t.c2)"!)"#,input: sort(col("c1") + min(col("c2"))),- // should be "c1" not t.c1expected: sort(col("c1") + Expr::Column(Column::new_unqualified("min(t.c2)")),),},TestCase {- desc: r#"avg(c3) --> "avg(t.c3)" as average (column *named* "avg(t.c3)", aliased)"#,+ desc: r#"avg(c3) --> "average" (column *named* "average", from alias)"#,input: sort(avg(col("c3"))),- expected: sort(- Expr::Column(Column::new_unqualified("avg(t.c3)")).alias("average"),- ),+ expected: sort(col("average")),},];@@ -261,6 +257,202 @@ mod test {}}+ /// When an aggregate is aliased in the projection,+ /// ORDER BY on the original aggregate expression should resolve to+ /// a Column reference using the alias name — not leak the inner+ /// Alias expression node or resolve to a descendant subtree.+ #[test]+ fn rewrite_sort_resolves_alias_to_column_ref() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![min(col("c2")), max(col("c3"))])+ .unwrap()+ .project(vec![+ col("c1"),+ min(col("c2")).alias("min_val"),+ max(col("c3")).alias("max_val"),+ ])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![+ TestCase {+ desc: "min(c2) with alias 'min_val' should resolve to col(min_val)",+ input: sort(min(col("c2"))),+ expected: sort(col("min_val")),+ },+ TestCase {+ desc: "max(c3) with alias 'max_val' should resolve to col(max_val)",+ input: sort(max(col("c3"))),+ expected: sort(col("max_val")),+ },+ ];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn composite_proj_expr_containing_sort_col_as_subexpr() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![min(col("c2")), max(col("c3"))])+ .unwrap()+ .project(vec![+ col("c1"),+ (min(col("c2")) + max(col("c3"))).alias("range"),+ min(col("c2")).alias("min_val"),+ max(col("c3")).alias("max_val"),+ ])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![+ TestCase {+ desc: "sort by min(c2) should resolve to col(min_val), not col(range)",+ input: sort(min(col("c2"))),+ expected: sort(col("min_val")),+ },+ TestCase {+ desc: "sort by max(c3) should resolve to col(max_val), not col(range)",+ input: sort(max(col("c3"))),+ expected: sort(col("max_val")),+ },+ ];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn composite_before_standalone_should_not_shadow() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![min(col("c2")), max(col("c2"))])+ .unwrap()+ .project(vec![+ col("c1"),+ (min(col("c2")) + max(col("c2"))).alias("combined"),+ min(col("c2")),+ ])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![TestCase {+ desc: "sort by min(c2) should resolve to col(min(t.c2)), not col(combined)",+ input: sort(min(col("c2"))),+ expected: sort(Expr::Column(Column::new_unqualified("min(t.c2)"))),+ }];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn duplicate_aggregate_in_multiple_proj_exprs() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![min(col("c2"))])+ .unwrap()+ .project(vec![+ col("c1"),+ min(col("c2")).alias("first_alias"),+ min(col("c2")).alias("second_alias"),+ ])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![TestCase {+ desc: "sort by min(c2) with two aliases picks first_alias",+ input: sort(min(col("c2"))),+ expected: sort(col("first_alias")),+ }];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn sort_agg_not_in_select_with_aliased_aggs() {+ let plan = make_input()+ .aggregate(+ vec![col("c1")],+ vec![min(col("c2")), max(col("c3")), sum(col("c3"))],+ )+ .unwrap()+ .project(vec![+ col("c1"),+ min(col("c2")).alias("min_val"),+ max(col("c3")).alias("max_val"),+ ])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![TestCase {+ desc: "sort by sum(c3) not in projection should not be rewritten",+ input: sort(sum(col("c3"))),+ expected: sort(sum(col("c3"))),+ }];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn cast_on_aliased_aggregate() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![min(col("c2"))])+ .unwrap()+ .project(vec![col("c1"), min(col("c2")).alias("min_val")])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![+ TestCase {+ desc: "CAST on aliased aggregate should preserve cast and resolve alias",+ input: sort(cast(min(col("c2")), DataType::Int64)),+ expected: sort(cast(col("min_val"), DataType::Int64)),+ },+ TestCase {+ desc: "TryCast on aliased aggregate should preserve try_cast and resolve alias",+ input: sort(try_cast(min(col("c2")), DataType::Int64)),+ expected: sort(try_cast(col("min_val"), DataType::Int64)),+ },+ ];++ for case in cases {+ case.run(&plan)+ }+ }++ #[test]+ fn count_star_with_alias() {+ let plan = make_input()+ .aggregate(vec![col("c1")], vec![count(lit(1))])+ .unwrap()+ .project(vec![col("c1"), count(lit(1)).alias("cnt")])+ .unwrap()+ .build()+ .unwrap();++ let cases = vec![TestCase {+ desc: "sort by count(1) should resolve to cnt alias",+ input: sort(count(lit(1))),+ expected: sort(col("cnt")),+ }];++ for case in cases {+ case.run(&plan)+ }+ }+#[test]fn preserve_cast() {let plan = make_input()@@ -275,12 +467,12 @@ mod test {TestCase {desc: "Cast is preserved by rewrite_sort_cols_by_aggs",input: sort(cast(col("c2"), DataType::Int64)),- expected: sort(cast(col("c2").alias("c2"), DataType::Int64)),+ expected: sort(cast(col("c2"), DataType::Int64)),},TestCase {desc: "TryCast is preserved by rewrite_sort_cols_by_aggs",input: sort(try_cast(col("c2"), DataType::Int64)),- expected: sort(try_cast(col("c2").alias("c2"), DataType::Int64)),+ expected: sort(try_cast(col("c2"), DataType::Int64)),},];diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex edf4b9ef79..7e291afa04 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -1056,13 +1056,16 @@ impl<S: ContextProvider> SqlToRel<'_, S> {.iter().find_map(|select_expr| {// Only consider aliased expressions- if let Expr::Alias(alias) = select_expr- && alias.expr.as_ref() == &rewritten_expr- {- // Use the alias name- return Some(Expr::Column(Column::new_unqualified(- alias.name.clone(),- )));+ if let Expr::Alias(alias) = select_expr {+ let rewritten_unaliased = match &rewritten_expr {+ Expr::Alias(a) => a.expr.as_ref(),+ other => other,+ };+ if alias.expr.as_ref() == rewritten_unaliased {+ return Some(Expr::Column(Column::new_unqualified(+ alias.name.clone(),+ )));+ }}None})diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.sltindex 881e49cdeb..e14d28d5ef 100644--- a/datafusion/sqllogictest/test_files/clickbench.slt+++ b/datafusion/sqllogictest/test_files/clickbench.slt@@ -205,7 +205,7 @@ query TTEXPLAIN SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;----logical_plan-01)Sort: count(*) AS count(*) DESC NULLS FIRST+01)Sort: count(*) DESC NULLS FIRST02)--Projection: hits.AdvEngineID, count(Int64(1)) AS count(*)03)----Aggregate: groupBy=[[hits.AdvEngineID]], aggr=[[count(Int64(1))]]04)------SubqueryAlias: hits@@ -431,7 +431,7 @@ query TTEXPLAIN SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10;----logical_plan-01)Sort: count(*) AS count(*) DESC NULLS FIRST, fetch=10+01)Sort: count(*) DESC NULLS FIRST, fetch=1002)--Projection: hits.UserID, count(Int64(1)) AS count(*)03)----Aggregate: groupBy=[[hits.UserID]], aggr=[[count(Int64(1))]]04)------SubqueryAlias: hits@@ -459,7 +459,7 @@ query TTEXPLAIN SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10;----logical_plan-01)Sort: count(*) AS count(*) DESC NULLS FIRST, fetch=10+01)Sort: count(*) DESC NULLS FIRST, fetch=1002)--Projection: hits.UserID, hits.SearchPhrase, count(Int64(1)) AS count(*)03)----Aggregate: groupBy=[[hits.UserID, hits.SearchPhrase]], aggr=[[count(Int64(1))]]04)------SubqueryAlias: hits@@ -514,7 +514,7 @@ query TTEXPLAIN SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10;----logical_plan-01)Sort: count(*) AS count(*) DESC NULLS FIRST, fetch=10+01)Sort: count(*) DESC NULLS FIRST, fetch=1002)--Projection: hits.UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)) AS m, hits.SearchPhrase, count(Int64(1)) AS count(*)03)----Aggregate: groupBy=[[hits.UserID, date_part(Utf8("MINUTE"), to_timestamp_seconds(hits.EventTime)), hits.SearchPhrase]], aggr=[[count(Int64(1))]]04)------SubqueryAlias: hitsdiff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 7c857cae36..892a42ad61 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -471,6 +471,54 @@ select column1 from foo order by column2 % 2, column2;35+# ORDER BY aggregate expression that is aliased in SELECT+query II+select column1, min(column2) as min_val from foo group by column1 order by min(column2);+----+1 2+3 4+5 6++# ORDER BY aggregate with alias, using DESC+query II rowsort+select column1, count(*) as cnt from foo group by column1 order by count(*) desc;+----+1 1+3 1+5 1++# ORDER BY aggregate not in SELECT, while other aggregates in SELECT are aliased+query I+select column1 from foo group by column1 order by max(column2);+----+1+3+5++# SELECT has composite expression containing the aggregate, plus standalone alias+query III+select column1, min(column2) + max(column2) as range_val, min(column2) as min_val from foo group by column1 order by min(column2);+----+1 4 2+3 8 4+5 12 6++# ORDER BY aggregate that matches multiple aliased SELECT expressions+query III+select column1, min(column2) as first_min, min(column2) as second_min from foo group by column1 order by min(column2);+----+1 2 2+3 4 4+5 6 6++# ORDER BY with CAST on aliased aggregate+query II+select column1, min(column2) as min_val from foo group by column1 order by CAST(min(column2) AS BIGINT);+----+1 2+3 4+5 6+# Cleanupstatement okdrop table foo;