Skip to content

Apache DataFusion SQL Query Engine

Fix error planning aggregates with duplicated names in select list (#18831)

Trent Haucke4dcf0c85611
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index 42013a76a8..fa9337a30a 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -41,7 +41,7 @@ use datafusion_expr::utils::{
};
use datafusion_expr::{
Aggregate, Expr, Filter, GroupingSet, LogicalPlan, LogicalPlanBuilder,
- LogicalPlanBuilderOptions, Partitioning,
+ LogicalPlanBuilderOptions, Partitioning, SortExpr,
};
use indexmap::IndexMap;
@@ -51,6 +51,21 @@ use sqlparser::ast::{
};
use sqlparser::ast::{NamedWindowDefinition, Select, SelectItem, TableWithJoins};
+/// Result of the `aggregate` function, containing the aggregate plan and
+/// rewritten expressions that reference the aggregate output columns.
+struct AggregatePlanResult {
+ /// The aggregate logical plan
+ plan: LogicalPlan,
+ /// SELECT expressions rewritten to reference aggregate output columns
+ select_exprs: Vec<Expr>,
+ /// HAVING expression rewritten to reference aggregate output columns
+ having_expr: Option<Expr>,
+ /// QUALIFY expression rewritten to reference aggregate output columns
+ qualify_expr: Option<Expr>,
+ /// ORDER BY expressions rewritten to reference aggregate output columns
+ order_by_exprs: Vec<SortExpr>,
+}
+
impl<S: ContextProvider> SqlToRel<'_, S> {
/// Generate a logic plan from an SQL select
pub(super) fn select_to_plan(
@@ -219,33 +234,53 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
.transpose()?;
// The outer expressions we will search through for aggregates.
- // Aggregates may be sourced from the SELECT list or from the HAVING expression.
- let aggr_expr_haystack = select_exprs
- .iter()
- .chain(having_expr_opt.iter())
- .chain(qualify_expr_opt.iter());
- // All of the aggregate expressions (deduplicated).
- let aggr_exprs = find_aggregate_exprs(aggr_expr_haystack);
+ // First, find aggregates in SELECT, HAVING, and QUALIFY
+ let select_having_qualify_aggrs = find_aggregate_exprs(
+ select_exprs
+ .iter()
+ .chain(having_expr_opt.iter())
+ .chain(qualify_expr_opt.iter()),
+ );
+
+ // Find aggregates in ORDER BY
+ let order_by_aggrs = find_aggregate_exprs(order_by_rex.iter().map(|s| &s.expr));
+
+ // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY aggregates
+ // that aren't already in SELECT/HAVING/QUALIFY
+ let mut aggr_exprs = select_having_qualify_aggrs;
+ for order_by_aggr in order_by_aggrs {
+ if !aggr_exprs.iter().any(|e| e == &order_by_aggr) {
+ aggr_exprs.push(order_by_aggr);
+ }
+ }
// Process group by, aggregation or having
- let (
+ let AggregatePlanResult {
plan,
- mut select_exprs_post_aggr,
- having_expr_post_aggr,
- qualify_expr_post_aggr,
- ) = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {
+ select_exprs: mut select_exprs_post_aggr,
+ having_expr: having_expr_post_aggr,
+ qualify_expr: qualify_expr_post_aggr,
+ order_by_exprs: order_by_rex,
+ } = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {
self.aggregate(
&base_plan,
&select_exprs,
having_expr_opt.as_ref(),
qualify_expr_opt.as_ref(),
+ &order_by_rex,
&group_by_exprs,
&aggr_exprs,
)?
} else {
match having_expr_opt {
Some(having_expr) => return plan_err!("HAVING clause references: {having_expr} must appear in the GROUP BY clause or be used in an aggregate function"),
- None => (base_plan.clone(), select_exprs.clone(), having_expr_opt, qualify_expr_opt)
+ None => AggregatePlanResult {
+ plan: base_plan.clone(),
+ select_exprs: select_exprs.clone(),
+ having_expr: having_expr_opt,
+ qualify_expr: qualify_expr_opt,
+ order_by_exprs: order_by_rex,
+ }
}
};
@@ -366,7 +401,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
plan
};
- self.order_by(plan, order_by_rex)
+ let plan = self.order_by(plan, order_by_rex)?;
+ Ok(plan)
}
/// Try converting Expr(Unnest(Expr)) to Projection/Unnest/Projection
@@ -872,16 +908,19 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
/// the aggregate
/// * `qualify_expr_post_aggr` - The "qualify" expression rewritten to reference a column from
/// the aggregate
- #[allow(clippy::type_complexity)]
+ /// * `order_by_post_aggr` - The ORDER BY expressions rewritten to reference columns from
+ /// the aggregate
+ #[allow(clippy::too_many_arguments)]
fn aggregate(
&self,
input: &LogicalPlan,
select_exprs: &[Expr],
having_expr_opt: Option<&Expr>,
qualify_expr_opt: Option<&Expr>,
+ order_by_exprs: &[SortExpr],
group_by_exprs: &[Expr],
aggr_exprs: &[Expr],
- ) -> Result<(LogicalPlan, Vec<Expr>, Option<Expr>, Option<Expr>)> {
+ ) -> Result<AggregatePlanResult> {
// create the aggregate plan
let options =
LogicalPlanBuilderOptions::new().with_add_implicit_group_by_exprs(true);
@@ -988,12 +1027,65 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
None
};
- Ok((
+ // Rewrite the ORDER BY expressions to use the columns produced by the
+ // aggregation. If an ORDER BY expression matches a SELECT expression
+ // (ignoring aliases), use the SELECT's output column name to avoid
+ // duplication when the SELECT expression has an alias.
+ let order_by_post_aggr = order_by_exprs
+ .iter()
+ .map(|sort_expr| {
+ let rewritten_expr =
+ rebase_expr(&sort_expr.expr, &aggr_projection_exprs, input)?;
+
+ // Check if this ORDER BY expression matches any aliased SELECT expression
+ // If so, use the SELECT's alias instead of the raw expression
+ let final_expr = select_exprs_post_aggr
+ .iter()
+ .find_map(|select_expr| {
+ // Only consider aliased expressions
+ if let Expr::Alias(alias) = select_expr {
+ if alias.expr.as_ref() == &rewritten_expr {
+ // Use the alias name
+ return Some(Expr::Column(alias.name.clone().into()));
+ }
+ }
+ None
+ })
+ .unwrap_or(rewritten_expr);
+
+ Ok(sort_expr.with_expr(final_expr))
+ })
+ .collect::<Result<Vec<SortExpr>>>()?;
+
+ let all_valid_exprs: Vec<Expr> = column_exprs_post_aggr
+ .iter()
+ .cloned()
+ .chain(select_exprs_post_aggr.iter().filter_map(|e| {
+ if let Expr::Alias(alias) = e {
+ Some(Expr::Column(alias.name.clone().into()))
+ } else {
+ None
+ }
+ }))
+ .collect();
+
+ let order_by_exprs_only: Vec<Expr> =
+ order_by_post_aggr.iter().map(|s| s.expr.clone()).collect();
+ check_columns_satisfy_exprs(
+ &all_valid_exprs,
+ &order_by_exprs_only,
+ CheckColumnsSatisfyExprsPurpose::Aggregate(
+ CheckColumnsMustReferenceAggregatePurpose::OrderBy,
+ ),
+ )?;
+
+ Ok(AggregatePlanResult {
plan,
- select_exprs_post_aggr,
- having_expr_post_aggr,
- qualify_expr_post_aggr,
- ))
+ select_exprs: select_exprs_post_aggr,
+ having_expr: having_expr_post_aggr,
+ qualify_expr: qualify_expr_post_aggr,
+ order_by_exprs: order_by_post_aggr,
+ })
}
// If the projection is done over a named window, that window
diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs
index 91ab2e003c..571d19db2e 100644
--- a/datafusion/sql/src/utils.rs
+++ b/datafusion/sql/src/utils.rs
@@ -97,6 +97,7 @@ pub(crate) enum CheckColumnsMustReferenceAggregatePurpose {
Projection,
Having,
Qualify,
+ OrderBy,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -116,6 +117,9 @@ impl CheckColumnsSatisfyExprsPurpose {
Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::Qualify) => {
"Column in QUALIFY must be in GROUP BY or an aggregate function"
}
+ Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::OrderBy) => {
+ "Column in ORDER BY must be in GROUP BY or an aggregate function"
+ }
}
}
diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs
index 5f76afb763..1c7813902f 100644
--- a/datafusion/sql/tests/cases/plan_to_sql.rs
+++ b/datafusion/sql/tests/cases/plan_to_sql.rs
@@ -334,9 +334,7 @@ fn roundtrip_statement_with_dialect_1() -> Result<(), DataFusionError> {
sql: "select min(ta.j1_id) as j1_min from j1 ta order by min(ta.j1_id) limit 10;",
parser_dialect: MySqlDialect {},
unparser_dialect: UnparserMySqlDialect {},
- // top projection sort gets derived into a subquery
- // for MySQL, this subquery needs an alias
- expected: @"SELECT `j1_min` FROM (SELECT min(`ta`.`j1_id`) AS `j1_min`, min(`ta`.`j1_id`) FROM `j1` AS `ta` ORDER BY min(`ta`.`j1_id`) ASC) AS `derived_sort` LIMIT 10",
+ expected: @"SELECT min(`ta`.`j1_id`) AS `j1_min` FROM `j1` AS `ta` ORDER BY `j1_min` ASC LIMIT 10",
);
Ok(())
}
@@ -347,9 +345,7 @@ fn roundtrip_statement_with_dialect_2() -> Result<(), DataFusionError> {
sql: "select min(ta.j1_id) as j1_min from j1 ta order by min(ta.j1_id) limit 10;",
parser_dialect: GenericDialect {},
unparser_dialect: UnparserDefaultDialect {},
- // top projection sort still gets derived into a subquery in default dialect
- // except for the default dialect, the subquery is left non-aliased
- expected: @"SELECT j1_min FROM (SELECT min(ta.j1_id) AS j1_min, min(ta.j1_id) FROM j1 AS ta ORDER BY min(ta.j1_id) ASC NULLS LAST) LIMIT 10",
+ expected: @"SELECT min(ta.j1_id) AS j1_min FROM j1 AS ta ORDER BY j1_min ASC NULLS LAST LIMIT 10",
);
Ok(())
}
@@ -360,7 +356,7 @@ fn roundtrip_statement_with_dialect_3() -> Result<(), DataFusionError> {
sql: "select min(ta.j1_id) as j1_min, max(tb.j1_max) from j1 ta, (select distinct max(ta.j1_id) as j1_max from j1 ta order by max(ta.j1_id)) tb order by min(ta.j1_id) limit 10;",
parser_dialect: MySqlDialect {},
unparser_dialect: UnparserMySqlDialect {},
- expected: @"SELECT `j1_min`, `max(tb.j1_max)` FROM (SELECT min(`ta`.`j1_id`) AS `j1_min`, max(`tb`.`j1_max`), min(`ta`.`j1_id`) FROM `j1` AS `ta` CROSS JOIN (SELECT `j1_max` FROM (SELECT DISTINCT max(`ta`.`j1_id`) AS `j1_max` FROM `j1` AS `ta`) AS `derived_distinct`) AS `tb` ORDER BY min(`ta`.`j1_id`) ASC) AS `derived_sort` LIMIT 10",
+ expected: @"SELECT min(`ta`.`j1_id`) AS `j1_min`, max(`tb`.`j1_max`) FROM `j1` AS `ta` CROSS JOIN (SELECT DISTINCT max(`ta`.`j1_id`) AS `j1_max` FROM `j1` AS `ta`) AS `tb` ORDER BY `j1_min` ASC LIMIT 10",
);
Ok(())
}
@@ -1909,7 +1905,7 @@ fn test_complex_order_by_with_grouping() -> Result<()> {
}, {
assert_snapshot!(
sql,
- @r#"SELECT j1.j1_id, j1.j1_string, lochierarchy FROM (SELECT j1.j1_id, j1.j1_string, (grouping(j1.j1_id) + grouping(j1.j1_string)) AS lochierarchy, grouping(j1.j1_string), grouping(j1.j1_id) FROM j1 GROUP BY ROLLUP (j1.j1_id, j1.j1_string) ORDER BY (grouping(j1.j1_id) + grouping(j1.j1_string)) DESC NULLS FIRST, CASE WHEN ((grouping(j1.j1_id) + grouping(j1.j1_string)) = 0) THEN j1.j1_id END ASC NULLS LAST) LIMIT 100"#
+ @r#"SELECT j1.j1_id, j1.j1_string, lochierarchy FROM (SELECT j1.j1_id, j1.j1_string, (grouping(j1.j1_id) + grouping(j1.j1_string)) AS lochierarchy, grouping(j1.j1_string), grouping(j1.j1_id) FROM j1 GROUP BY ROLLUP (j1.j1_id, j1.j1_string) ORDER BY lochierarchy DESC NULLS FIRST, CASE WHEN ((grouping(j1.j1_id) + grouping(j1.j1_string)) = 0) THEN j1.j1_id END ASC NULLS LAST) LIMIT 100"#
);
});
diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs
index 2d62fd9c0c..7615891be3 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -3261,6 +3261,55 @@ Sort: avg(person.age) + avg(person.age) ASC NULLS LAST
);
}
+#[test]
+fn select_groupby_orderby_aggregate_on_non_selected_column() {
+ let sql = "SELECT state FROM person GROUP BY state ORDER BY MIN(age)";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+Projection: person.state
+ Sort: min(person.age) ASC NULLS LAST
+ Projection: person.state, min(person.age)
+ Aggregate: groupBy=[[person.state]], aggr=[[min(person.age)]]
+ TableScan: person
+"#
+ );
+}
+
+#[test]
+fn select_groupby_orderby_multiple_aggregates_on_non_selected_columns() {
+ let sql =
+ "SELECT state FROM person GROUP BY state ORDER BY MIN(age), MAX(salary) DESC";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+Projection: person.state
+ Sort: min(person.age) ASC NULLS LAST, max(person.salary) DESC NULLS FIRST
+ Projection: person.state, min(person.age), max(person.salary)
+ Aggregate: groupBy=[[person.state]], aggr=[[min(person.age), max(person.salary)]]
+ TableScan: person
+"#
+ );
+}
+
+#[test]
+fn select_groupby_orderby_aggregate_on_non_selected_column_original_issue() {
+ let sql = "SELECT id FROM person GROUP BY id ORDER BY min(age)";
+ let plan = logical_plan(sql).unwrap();
+ assert_snapshot!(
+ plan,
+ @r#"
+Projection: person.id
+ Sort: min(person.age) ASC NULLS LAST
+ Projection: person.id, min(person.age)
+ Aggregate: groupBy=[[person.id]], aggr=[[min(person.age)]]
+ TableScan: person
+"#
+ );
+}
+
fn logical_plan(sql: &str) -> Result<LogicalPlan> {
logical_plan_with_options(sql, ParserOptions::default())
}
@@ -3830,12 +3879,11 @@ fn order_by_unaliased_name() {
assert_snapshot!(
plan,
@r#"
-Projection: z, q
- Sort: p.state ASC NULLS LAST
- Projection: p.state AS z, sum(p.age) AS q, p.state
- Aggregate: groupBy=[[p.state]], aggr=[[sum(p.age)]]
- SubqueryAlias: p
- TableScan: person
+Sort: z ASC NULLS LAST
+ Projection: p.state AS z, sum(p.age) AS q
+ Aggregate: groupBy=[[p.state]], aggr=[[sum(p.age)]]
+ SubqueryAlias: p
+ TableScan: person
"#
);
}
diff --git a/datafusion/sqllogictest/test_files/join.slt.part b/datafusion/sqllogictest/test_files/join.slt.part
index ea998f742d..92b6ed667c 100644
--- a/datafusion/sqllogictest/test_files/join.slt.part
+++ b/datafusion/sqllogictest/test_files/join.slt.part
@@ -1504,7 +1504,7 @@ drop table t0;
statement ok
create table t1(v1 int, v2 int);
-query error DataFusion error: Schema error: No field named tt1.v2. Valid fields are tt1.v1.
+query error DataFusion error: Error during planning: Column in ORDER BY must be in GROUP BY or an aggregate function
select v1 from t1 as tt1 natural join t1 as tt2 group by v1 order by v2;
statement ok
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 03ba12ee98..86650386a4 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -419,6 +419,31 @@ select column1 + column2 from foo group by column1, column2 ORDER BY column2 des
7
3
+# Test ordering by aggregate on non-selected column (issue #18683)
+# Previously failed with "Schema error: No field named foo.column2"
+query I
+select column1 from foo group by column1 order by min(column2);
+----
+1
+3
+5
+
+# Test ordering by aggregate expression on non-selected columns
+query I
+select column1 from foo group by column1 order by min(column2) + max(column2);
+----
+1
+3
+5
+
+# Test ordering by multiple aggregates on non-selected columns
+query I
+select column1 from foo group by column1 order by min(column2), max(column2);
+----
+1
+3
+5
+
# Test issue: https://github.com/apache/datafusion/issues/11549
query I
select column1 from foo order by log(column2);
@@ -1055,8 +1080,8 @@ SELECT SUM(column1) FROM foo ORDER BY SUM(column1)
----
16
-# Order by unprojected aggregate expressions is not supported
-query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression AggregateFunction
+# Order by unprojected aggregate expressions requires GROUP BY
+query error DataFusion error: Error during planning: Column in SELECT must be in GROUP BY or an aggregate function
SELECT column2 FROM foo ORDER BY SUM(column1)
statement ok