Apache DataFusion SQL Query Engine
Fix error planning aggregates with duplicated names in select list (#18831)
Trent Haucke4dcf0c85611
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex 42013a76a8..fa9337a30a 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -41,7 +41,7 @@ use datafusion_expr::utils::{};use datafusion_expr::{Aggregate, Expr, Filter, GroupingSet, LogicalPlan, LogicalPlanBuilder,- LogicalPlanBuilderOptions, Partitioning,+ LogicalPlanBuilderOptions, Partitioning, SortExpr,};use indexmap::IndexMap;@@ -51,6 +51,21 @@ use sqlparser::ast::{};use sqlparser::ast::{NamedWindowDefinition, Select, SelectItem, TableWithJoins};+/// Result of the `aggregate` function, containing the aggregate plan and+/// rewritten expressions that reference the aggregate output columns.+struct AggregatePlanResult {+ /// The aggregate logical plan+ plan: LogicalPlan,+ /// SELECT expressions rewritten to reference aggregate output columns+ select_exprs: Vec<Expr>,+ /// HAVING expression rewritten to reference aggregate output columns+ having_expr: Option<Expr>,+ /// QUALIFY expression rewritten to reference aggregate output columns+ qualify_expr: Option<Expr>,+ /// ORDER BY expressions rewritten to reference aggregate output columns+ order_by_exprs: Vec<SortExpr>,+}+impl<S: ContextProvider> SqlToRel<'_, S> {/// Generate a logic plan from an SQL selectpub(super) fn select_to_plan(@@ -219,33 +234,53 @@ impl<S: ContextProvider> SqlToRel<'_, S> {.transpose()?;// The outer expressions we will search through for aggregates.- // Aggregates may be sourced from the SELECT list or from the HAVING expression.- let aggr_expr_haystack = select_exprs- .iter()- .chain(having_expr_opt.iter())- .chain(qualify_expr_opt.iter());- // All of the aggregate expressions (deduplicated).- let aggr_exprs = find_aggregate_exprs(aggr_expr_haystack);+ // First, find aggregates in SELECT, HAVING, and QUALIFY+ let select_having_qualify_aggrs = find_aggregate_exprs(+ select_exprs+ .iter()+ .chain(having_expr_opt.iter())+ .chain(qualify_expr_opt.iter()),+ );++ // Find aggregates in ORDER BY+ let order_by_aggrs = find_aggregate_exprs(order_by_rex.iter().map(|s| &s.expr));++ // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY aggregates+ // that aren't already in SELECT/HAVING/QUALIFY+ let mut aggr_exprs = select_having_qualify_aggrs;+ for order_by_aggr in order_by_aggrs {+ if !aggr_exprs.iter().any(|e| e == &order_by_aggr) {+ aggr_exprs.push(order_by_aggr);+ }+ }// Process group by, aggregation or having- let (+ let AggregatePlanResult {plan,- mut select_exprs_post_aggr,- having_expr_post_aggr,- qualify_expr_post_aggr,- ) = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {+ select_exprs: mut select_exprs_post_aggr,+ having_expr: having_expr_post_aggr,+ qualify_expr: qualify_expr_post_aggr,+ order_by_exprs: order_by_rex,+ } = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {self.aggregate(&base_plan,&select_exprs,having_expr_opt.as_ref(),qualify_expr_opt.as_ref(),+ &order_by_rex,&group_by_exprs,&aggr_exprs,)?} else {match having_expr_opt {Some(having_expr) => return plan_err!("HAVING clause references: {having_expr} must appear in the GROUP BY clause or be used in an aggregate function"),- None => (base_plan.clone(), select_exprs.clone(), having_expr_opt, qualify_expr_opt)+ None => AggregatePlanResult {+ plan: base_plan.clone(),+ select_exprs: select_exprs.clone(),+ having_expr: having_expr_opt,+ qualify_expr: qualify_expr_opt,+ order_by_exprs: order_by_rex,+ }}};@@ -366,7 +401,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {plan};- self.order_by(plan, order_by_rex)+ let plan = self.order_by(plan, order_by_rex)?;+ Ok(plan)}/// Try converting Expr(Unnest(Expr)) to Projection/Unnest/Projection@@ -872,16 +908,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {/// the aggregate/// * `qualify_expr_post_aggr` - The "qualify" expression rewritten to reference a column from/// the aggregate- #[allow(clippy::type_complexity)]+ /// * `order_by_post_aggr` - The ORDER BY expressions rewritten to reference columns from+ /// the aggregate+ #[allow(clippy::too_many_arguments)]fn aggregate(&self,input: &LogicalPlan,select_exprs: &[Expr],having_expr_opt: Option<&Expr>,qualify_expr_opt: Option<&Expr>,+ order_by_exprs: &[SortExpr],group_by_exprs: &[Expr],aggr_exprs: &[Expr],- ) -> Result<(LogicalPlan, Vec<Expr>, Option<Expr>, Option<Expr>)> {+ ) -> Result<AggregatePlanResult> {// create the aggregate planlet options =LogicalPlanBuilderOptions::new().with_add_implicit_group_by_exprs(true);@@ -988,12 +1027,65 @@ impl<S: ContextProvider> SqlToRel<'_, S> {None};- Ok((+ // Rewrite the ORDER BY expressions to use the columns produced by the+ // aggregation. If an ORDER BY expression matches a SELECT expression+ // (ignoring aliases), use the SELECT's output column name to avoid+ // duplication when the SELECT expression has an alias.+ let order_by_post_aggr = order_by_exprs+ .iter()+ .map(|sort_expr| {+ let rewritten_expr =+ rebase_expr(&sort_expr.expr, &aggr_projection_exprs, input)?;++ // Check if this ORDER BY expression matches any aliased SELECT expression+ // If so, use the SELECT's alias instead of the raw expression+ let final_expr = select_exprs_post_aggr+ .iter()+ .find_map(|select_expr| {+ // Only consider aliased expressions+ if let Expr::Alias(alias) = select_expr {+ if alias.expr.as_ref() == &rewritten_expr {+ // Use the alias name+ return Some(Expr::Column(alias.name.clone().into()));+ }+ }+ None+ })+ .unwrap_or(rewritten_expr);++ Ok(sort_expr.with_expr(final_expr))+ })+ .collect::<Result<Vec<SortExpr>>>()?;++ let all_valid_exprs: Vec<Expr> = column_exprs_post_aggr+ .iter()+ .cloned()+ .chain(select_exprs_post_aggr.iter().filter_map(|e| {+ if let Expr::Alias(alias) = e {+ Some(Expr::Column(alias.name.clone().into()))+ } else {+ None+ }+ }))+ .collect();++ let order_by_exprs_only: Vec<Expr> =+ order_by_post_aggr.iter().map(|s| s.expr.clone()).collect();+ check_columns_satisfy_exprs(+ &all_valid_exprs,+ &order_by_exprs_only,+ CheckColumnsSatisfyExprsPurpose::Aggregate(+ CheckColumnsMustReferenceAggregatePurpose::OrderBy,+ ),+ )?;++ Ok(AggregatePlanResult {plan,- select_exprs_post_aggr,- having_expr_post_aggr,- qualify_expr_post_aggr,- ))+ select_exprs: select_exprs_post_aggr,+ having_expr: having_expr_post_aggr,+ qualify_expr: qualify_expr_post_aggr,+ order_by_exprs: order_by_post_aggr,+ })}// If the projection is done over a named window, that windowdiff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rsindex 91ab2e003c..571d19db2e 100644--- a/datafusion/sql/src/utils.rs+++ b/datafusion/sql/src/utils.rs@@ -97,6 +97,7 @@ pub(crate) enum CheckColumnsMustReferenceAggregatePurpose {Projection,Having,Qualify,+ OrderBy,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -116,6 +117,9 @@ impl CheckColumnsSatisfyExprsPurpose {Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::Qualify) => {"Column in QUALIFY must be in GROUP BY or an aggregate function"}+ Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::OrderBy) => {+ "Column in ORDER BY must be in GROUP BY or an aggregate function"+ }}}diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex 5f76afb763..1c7813902f 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -334,9 +334,7 @@ fn roundtrip_statement_with_dialect_1() -> Result<(), DataFusionError> {sql: "select min(ta.j1_id) as j1_min from j1 ta order by min(ta.j1_id) limit 10;",parser_dialect: MySqlDialect {},unparser_dialect: UnparserMySqlDialect {},- // top projection sort gets derived into a subquery- // for MySQL, this subquery needs an alias- expected: @"SELECT `j1_min` FROM (SELECT min(`ta`.`j1_id`) AS `j1_min`, min(`ta`.`j1_id`) FROM `j1` AS `ta` ORDER BY min(`ta`.`j1_id`) ASC) AS `derived_sort` LIMIT 10",+ expected: @"SELECT min(`ta`.`j1_id`) AS `j1_min` FROM `j1` AS `ta` ORDER BY `j1_min` ASC LIMIT 10",);Ok(())}@@ -347,9 +345,7 @@ fn roundtrip_statement_with_dialect_2() -> Result<(), DataFusionError> {sql: "select min(ta.j1_id) as j1_min from j1 ta order by min(ta.j1_id) limit 10;",parser_dialect: GenericDialect {},unparser_dialect: UnparserDefaultDialect {},- // top projection sort still gets derived into a subquery in default dialect- // except for the default dialect, the subquery is left non-aliased- expected: @"SELECT j1_min FROM (SELECT min(ta.j1_id) AS j1_min, min(ta.j1_id) FROM j1 AS ta ORDER BY min(ta.j1_id) ASC NULLS LAST) LIMIT 10",+ expected: @"SELECT min(ta.j1_id) AS j1_min FROM j1 AS ta ORDER BY j1_min ASC NULLS LAST LIMIT 10",);Ok(())}@@ -360,7 +356,7 @@ fn roundtrip_statement_with_dialect_3() -> Result<(), DataFusionError> {sql: "select min(ta.j1_id) as j1_min, max(tb.j1_max) from j1 ta, (select distinct max(ta.j1_id) as j1_max from j1 ta order by max(ta.j1_id)) tb order by min(ta.j1_id) limit 10;",parser_dialect: MySqlDialect {},unparser_dialect: UnparserMySqlDialect {},- expected: @"SELECT `j1_min`, `max(tb.j1_max)` FROM (SELECT min(`ta`.`j1_id`) AS `j1_min`, max(`tb`.`j1_max`), min(`ta`.`j1_id`) FROM `j1` AS `ta` CROSS JOIN (SELECT `j1_max` FROM (SELECT DISTINCT max(`ta`.`j1_id`) AS `j1_max` FROM `j1` AS `ta`) AS `derived_distinct`) AS `tb` ORDER BY min(`ta`.`j1_id`) ASC) AS `derived_sort` LIMIT 10",+ expected: @"SELECT min(`ta`.`j1_id`) AS `j1_min`, max(`tb`.`j1_max`) FROM `j1` AS `ta` CROSS JOIN (SELECT DISTINCT max(`ta`.`j1_id`) AS `j1_max` FROM `j1` AS `ta`) AS `tb` ORDER BY `j1_min` ASC LIMIT 10",);Ok(())}@@ -1909,7 +1905,7 @@ fn test_complex_order_by_with_grouping() -> Result<()> {}, {assert_snapshot!(sql,- @r#"SELECT j1.j1_id, j1.j1_string, lochierarchy FROM (SELECT j1.j1_id, j1.j1_string, (grouping(j1.j1_id) + grouping(j1.j1_string)) AS lochierarchy, grouping(j1.j1_string), grouping(j1.j1_id) FROM j1 GROUP BY ROLLUP (j1.j1_id, j1.j1_string) ORDER BY (grouping(j1.j1_id) + grouping(j1.j1_string)) DESC NULLS FIRST, CASE WHEN ((grouping(j1.j1_id) + grouping(j1.j1_string)) = 0) THEN j1.j1_id END ASC NULLS LAST) LIMIT 100"#+ @r#"SELECT j1.j1_id, j1.j1_string, lochierarchy FROM (SELECT j1.j1_id, j1.j1_string, (grouping(j1.j1_id) + grouping(j1.j1_string)) AS lochierarchy, grouping(j1.j1_string), grouping(j1.j1_id) FROM j1 GROUP BY ROLLUP (j1.j1_id, j1.j1_string) ORDER BY lochierarchy DESC NULLS FIRST, CASE WHEN ((grouping(j1.j1_id) + grouping(j1.j1_string)) = 0) THEN j1.j1_id END ASC NULLS LAST) LIMIT 100"#);});diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rsindex 2d62fd9c0c..7615891be3 100644--- a/datafusion/sql/tests/sql_integration.rs+++ b/datafusion/sql/tests/sql_integration.rs@@ -3261,6 +3261,55 @@ Sort: avg(person.age) + avg(person.age) ASC NULLS LAST);}+#[test]+fn select_groupby_orderby_aggregate_on_non_selected_column() {+ let sql = "SELECT state FROM person GROUP BY state ORDER BY MIN(age)";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r#"+Projection: person.state+ Sort: min(person.age) ASC NULLS LAST+ Projection: person.state, min(person.age)+ Aggregate: groupBy=[[person.state]], aggr=[[min(person.age)]]+ TableScan: person+"#+ );+}++#[test]+fn select_groupby_orderby_multiple_aggregates_on_non_selected_columns() {+ let sql =+ "SELECT state FROM person GROUP BY state ORDER BY MIN(age), MAX(salary) DESC";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r#"+Projection: person.state+ Sort: min(person.age) ASC NULLS LAST, max(person.salary) DESC NULLS FIRST+ Projection: person.state, min(person.age), max(person.salary)+ Aggregate: groupBy=[[person.state]], aggr=[[min(person.age), max(person.salary)]]+ TableScan: person+"#+ );+}++#[test]+fn select_groupby_orderby_aggregate_on_non_selected_column_original_issue() {+ let sql = "SELECT id FROM person GROUP BY id ORDER BY min(age)";+ let plan = logical_plan(sql).unwrap();+ assert_snapshot!(+ plan,+ @r#"+Projection: person.id+ Sort: min(person.age) ASC NULLS LAST+ Projection: person.id, min(person.age)+ Aggregate: groupBy=[[person.id]], aggr=[[min(person.age)]]+ TableScan: person+"#+ );+}+fn logical_plan(sql: &str) -> Result<LogicalPlan> {logical_plan_with_options(sql, ParserOptions::default())}@@ -3830,12 +3879,11 @@ fn order_by_unaliased_name() {assert_snapshot!(plan,@r#"-Projection: z, q- Sort: p.state ASC NULLS LAST- Projection: p.state AS z, sum(p.age) AS q, p.state- Aggregate: groupBy=[[p.state]], aggr=[[sum(p.age)]]- SubqueryAlias: p- TableScan: person+Sort: z ASC NULLS LAST+ Projection: p.state AS z, sum(p.age) AS q+ Aggregate: groupBy=[[p.state]], aggr=[[sum(p.age)]]+ SubqueryAlias: p+ TableScan: person"#);}diff --git a/datafusion/sqllogictest/test_files/join.slt.part b/datafusion/sqllogictest/test_files/join.slt.partindex ea998f742d..92b6ed667c 100644--- a/datafusion/sqllogictest/test_files/join.slt.part+++ b/datafusion/sqllogictest/test_files/join.slt.part@@ -1504,7 +1504,7 @@ drop table t0;statement okcreate table t1(v1 int, v2 int);-query error DataFusion error: Schema error: No field named tt1.v2. Valid fields are tt1.v1.+query error DataFusion error: Error during planning: Column in ORDER BY must be in GROUP BY or an aggregate functionselect v1 from t1 as tt1 natural join t1 as tt2 group by v1 order by v2;statement okdiff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 03ba12ee98..86650386a4 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -419,6 +419,31 @@ select column1 + column2 from foo group by column1, column2 ORDER BY column2 des73+# Test ordering by aggregate on non-selected column (issue #18683)+# Previously failed with "Schema error: No field named foo.column2"+query I+select column1 from foo group by column1 order by min(column2);+----+1+3+5++# Test ordering by aggregate expression on non-selected columns+query I+select column1 from foo group by column1 order by min(column2) + max(column2);+----+1+3+5++# Test ordering by multiple aggregates on non-selected columns+query I+select column1 from foo group by column1 order by min(column2), max(column2);+----+1+3+5+# Test issue: https://github.com/apache/datafusion/issues/11549query Iselect column1 from foo order by log(column2);@@ -1055,8 +1080,8 @@ SELECT SUM(column1) FROM foo ORDER BY SUM(column1)----16-# Order by unprojected aggregate expressions is not supported-query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression AggregateFunction+# Order by unprojected aggregate expressions requires GROUP BY+query error DataFusion error: Error during planning: Column in SELECT must be in GROUP BY or an aggregate functionSELECT column2 FROM foo ORDER BY SUM(column1)statement ok