Skip to content

Apache DataFusion SQL Query Engine

fix: unparser drops ORDER BY of ordered aggregates (#25803)

Zeren Wang2cac71327449
diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs
index 0f2cef167c..3fb349f5ed 100644
--- a/datafusion/sql/src/unparser/expr.rs
+++ b/datafusion/sql/src/unparser/expr.rs
@@ -408,18 +408,29 @@ impl Unparser<'_> {
..
} = &agg.params;
- // if this is a WITHIN GROUP aggregate, skip the prepended arg
- let (args_to_use, within_group) =
- if agg.func.supports_within_group_clause() && !order_by.is_empty() {
- let args_to_use = self.function_args_to_sql(&args[1..])?;
- let within_group = order_by
- .iter()
- .map(|sort_expr| self.sort_to_sql(sort_expr))
- .collect::<Result<Vec<ast::OrderByExpr>>>()?;
- (args_to_use, within_group)
+ // An aggregate's `order_by` is spelled one of two ways in SQL:
+ // `WITHIN GROUP (ORDER BY ..)` for ordered-set aggregates, and an
+ // `ORDER BY` clause inside the argument list for every other
+ // aggregate. Both map onto `params.order_by`.
+ let (args_to_use, within_group, clauses) = if order_by.is_empty() {
+ (self.function_args_to_sql(args)?, vec![], vec![])
+ } else {
+ let order_by_sql = order_by
+ .iter()
+ .map(|sort_expr| self.sort_to_sql(sort_expr))
+ .collect::<Result<Vec<ast::OrderByExpr>>>()?;
+ if agg.func.supports_within_group_clause() {
+ // Ordered-set aggregates carry the ordered value as their
+ // first argument, so skip it.
+ (self.function_args_to_sql(&args[1..])?, order_by_sql, vec![])
} else {
- (self.function_args_to_sql(args)?, Vec::new())
- };
+ (
+ self.function_args_to_sql(args)?,
+ vec![],
+ vec![ast::FunctionArgumentClause::OrderBy(order_by_sql)],
+ )
+ }
+ };
let filter = match filter {
Some(filter) => Some(Box::new(self.expr_to_sql_inner(filter)?)),
@@ -435,7 +446,7 @@ impl Unparser<'_> {
duplicate_treatment: distinct
.then_some(DuplicateTreatment::Distinct),
args: args_to_use,
- clauses: vec![],
+ clauses,
}),
filter,
null_treatment: None,
diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs
index de22321327..75a536d3ec 100644
--- a/datafusion/sql/tests/cases/plan_to_sql.rs
+++ b/datafusion/sql/tests/cases/plan_to_sql.rs
@@ -241,7 +241,14 @@ fn roundtrip_statement() -> Result<()> {
"SELECT left[1] FROM array",
"SELECT {a:1, b:2}",
"SELECT s.a FROM (SELECT {a:1, b:2} AS s)",
- "SELECT MAP {'a': 1, 'b': 2}"
+ "SELECT MAP {'a': 1, 'b': 2}",
+ // Ordered aggregates, both spellings of the ordering.
+ "SELECT first_name, last_value(age ORDER BY salary) FROM person GROUP BY first_name",
+ "SELECT first_name, first_value(age ORDER BY salary DESC) FROM person GROUP BY first_name",
+ "SELECT first_name, array_agg(age ORDER BY salary) FROM person GROUP BY first_name",
+ "SELECT first_name, array_agg(DISTINCT age ORDER BY salary, id) FROM person GROUP BY first_name",
+ "SELECT first_name, string_agg(CAST(age AS VARCHAR), ',' ORDER BY salary) FROM person GROUP BY first_name",
+ "SELECT first_name, percentile_cont(0.5) WITHIN GROUP (ORDER BY age) FROM person GROUP BY first_name",
];
// For each test sql string, we transform as follows:
@@ -261,6 +268,21 @@ fn roundtrip_statement() -> Result<()> {
.with_aggregate_function(sum_udaf())
.with_aggregate_function(count_udaf())
.with_aggregate_function(max_udaf())
+ .with_aggregate_function(
+ datafusion_functions_aggregate::array_agg::array_agg_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::first_last::first_value_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::first_last::last_value_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::string_agg::string_agg_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::percentile_cont::percentile_cont_udaf(),
+ )
.with_expr_planner(Arc::new(CoreFunctionPlanner::default()))
.with_expr_planner(Arc::new(NestedFunctionPlanner))
.with_expr_planner(Arc::new(FieldAccessPlanner));
@@ -343,6 +365,18 @@ macro_rules! roundtrip_statement_with_dialect_helper {
.with_aggregate_function(
datafusion_functions_aggregate::percentile_cont::percentile_cont_udaf(),
)
+ .with_aggregate_function(
+ datafusion_functions_aggregate::array_agg::array_agg_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::first_last::first_value_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::first_last::last_value_udaf(),
+ )
+ .with_aggregate_function(
+ datafusion_functions_aggregate::string_agg::string_agg_udaf(),
+ )
.with_expr_planner(Arc::new(CoreFunctionPlanner::default()))
.with_expr_planner(Arc::new(NestedFunctionPlanner))
.with_expr_planner(Arc::new(FieldAccessPlanner));
@@ -4611,6 +4645,45 @@ fn roundtrip_approx_percentile_cont_within_group_with_centroids()
Ok(())
}
+/// Ordered aggregates spell their ordering in one of two ways: an argument-list
+/// clause (`array_agg(x ORDER BY y)`) or a `WITHIN GROUP` clause. The
+/// `roundtrip_*_within_group` tests above cover the `WITHIN GROUP` variant;
+/// this covers the argument-list variant.
+#[test]
+fn roundtrip_ordered_aggregate_order_by() -> Result<(), DataFusionError> {
+ roundtrip_statement_with_dialect_helper!(
+ sql: "SELECT first_name, last_value(age ORDER BY salary) FROM person GROUP BY first_name",
+ parser_dialect: GenericDialect {},
+ unparser_dialect: UnparserDefaultDialect {},
+ expected: @"SELECT person.first_name, last_value(person.age ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",
+ );
+ roundtrip_statement_with_dialect_helper!(
+ sql: "SELECT first_name, first_value(age ORDER BY salary DESC) FROM person GROUP BY first_name",
+ parser_dialect: GenericDialect {},
+ unparser_dialect: UnparserDefaultDialect {},
+ expected: @"SELECT person.first_name, first_value(person.age ORDER BY person.salary DESC NULLS FIRST) FROM person GROUP BY person.first_name",
+ );
+ roundtrip_statement_with_dialect_helper!(
+ sql: "SELECT first_name, array_agg(age ORDER BY salary) FROM person GROUP BY first_name",
+ parser_dialect: GenericDialect {},
+ unparser_dialect: UnparserDefaultDialect {},
+ expected: @"SELECT person.first_name, array_agg(person.age ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",
+ );
+ roundtrip_statement_with_dialect_helper!(
+ sql: "SELECT first_name, string_agg(CAST(age AS VARCHAR), ',' ORDER BY salary) FROM person GROUP BY first_name",
+ parser_dialect: GenericDialect {},
+ unparser_dialect: UnparserDefaultDialect {},
+ expected: @"SELECT person.first_name, string_agg(CAST(person.age AS VARCHAR), ',' ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",
+ );
+ roundtrip_statement_with_dialect_helper!(
+ sql: "SELECT first_name, array_agg(DISTINCT age ORDER BY salary, id) FROM person GROUP BY first_name",
+ parser_dialect: GenericDialect {},
+ unparser_dialect: UnparserDefaultDialect {},
+ expected: @"SELECT person.first_name, array_agg(DISTINCT person.age ORDER BY person.salary ASC NULLS LAST, person.id ASC NULLS LAST) FROM person GROUP BY person.first_name",
+ );
+ Ok(())
+}
+
#[test]
fn snowflake_flatten_multiple_unnest_cross_join() -> Result<(), DataFusionError> {
// Realistic Snowflake pattern: