Apache DataFusion SQL Query Engine
fix: unparser drops ORDER BY of ordered aggregates (#25803)
Zeren Wang2cac71327449
diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex 0f2cef167c..3fb349f5ed 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -408,18 +408,29 @@ impl Unparser<'_> {..} = &agg.params;- // if this is a WITHIN GROUP aggregate, skip the prepended arg- let (args_to_use, within_group) =- if agg.func.supports_within_group_clause() && !order_by.is_empty() {- let args_to_use = self.function_args_to_sql(&args[1..])?;- let within_group = order_by- .iter()- .map(|sort_expr| self.sort_to_sql(sort_expr))- .collect::<Result<Vec<ast::OrderByExpr>>>()?;- (args_to_use, within_group)+ // An aggregate's `order_by` is spelled one of two ways in SQL:+ // `WITHIN GROUP (ORDER BY ..)` for ordered-set aggregates, and an+ // `ORDER BY` clause inside the argument list for every other+ // aggregate. Both map onto `params.order_by`.+ let (args_to_use, within_group, clauses) = if order_by.is_empty() {+ (self.function_args_to_sql(args)?, vec![], vec![])+ } else {+ let order_by_sql = order_by+ .iter()+ .map(|sort_expr| self.sort_to_sql(sort_expr))+ .collect::<Result<Vec<ast::OrderByExpr>>>()?;+ if agg.func.supports_within_group_clause() {+ // Ordered-set aggregates carry the ordered value as their+ // first argument, so skip it.+ (self.function_args_to_sql(&args[1..])?, order_by_sql, vec![])} else {- (self.function_args_to_sql(args)?, Vec::new())- };+ (+ self.function_args_to_sql(args)?,+ vec![],+ vec![ast::FunctionArgumentClause::OrderBy(order_by_sql)],+ )+ }+ };let filter = match filter {Some(filter) => Some(Box::new(self.expr_to_sql_inner(filter)?)),@@ -435,7 +446,7 @@ impl Unparser<'_> {duplicate_treatment: distinct.then_some(DuplicateTreatment::Distinct),args: args_to_use,- clauses: vec![],+ clauses,}),filter,null_treatment: None,diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex de22321327..75a536d3ec 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -241,7 +241,14 @@ fn roundtrip_statement() -> Result<()> {"SELECT left[1] FROM array","SELECT {a:1, b:2}","SELECT s.a FROM (SELECT {a:1, b:2} AS s)",- "SELECT MAP {'a': 1, 'b': 2}"+ "SELECT MAP {'a': 1, 'b': 2}",+ // Ordered aggregates, both spellings of the ordering.+ "SELECT first_name, last_value(age ORDER BY salary) FROM person GROUP BY first_name",+ "SELECT first_name, first_value(age ORDER BY salary DESC) FROM person GROUP BY first_name",+ "SELECT first_name, array_agg(age ORDER BY salary) FROM person GROUP BY first_name",+ "SELECT first_name, array_agg(DISTINCT age ORDER BY salary, id) FROM person GROUP BY first_name",+ "SELECT first_name, string_agg(CAST(age AS VARCHAR), ',' ORDER BY salary) FROM person GROUP BY first_name",+ "SELECT first_name, percentile_cont(0.5) WITHIN GROUP (ORDER BY age) FROM person GROUP BY first_name",];// For each test sql string, we transform as follows:@@ -261,6 +268,21 @@ fn roundtrip_statement() -> Result<()> {.with_aggregate_function(sum_udaf()).with_aggregate_function(count_udaf()).with_aggregate_function(max_udaf())+ .with_aggregate_function(+ datafusion_functions_aggregate::array_agg::array_agg_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::first_last::first_value_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::first_last::last_value_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::string_agg::string_agg_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::percentile_cont::percentile_cont_udaf(),+ ).with_expr_planner(Arc::new(CoreFunctionPlanner::default())).with_expr_planner(Arc::new(NestedFunctionPlanner)).with_expr_planner(Arc::new(FieldAccessPlanner));@@ -343,6 +365,18 @@ macro_rules! roundtrip_statement_with_dialect_helper {.with_aggregate_function(datafusion_functions_aggregate::percentile_cont::percentile_cont_udaf(),)+ .with_aggregate_function(+ datafusion_functions_aggregate::array_agg::array_agg_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::first_last::first_value_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::first_last::last_value_udaf(),+ )+ .with_aggregate_function(+ datafusion_functions_aggregate::string_agg::string_agg_udaf(),+ ).with_expr_planner(Arc::new(CoreFunctionPlanner::default())).with_expr_planner(Arc::new(NestedFunctionPlanner)).with_expr_planner(Arc::new(FieldAccessPlanner));@@ -4611,6 +4645,45 @@ fn roundtrip_approx_percentile_cont_within_group_with_centroids()Ok(())}+/// Ordered aggregates spell their ordering in one of two ways: an argument-list+/// clause (`array_agg(x ORDER BY y)`) or a `WITHIN GROUP` clause. The+/// `roundtrip_*_within_group` tests above cover the `WITHIN GROUP` variant;+/// this covers the argument-list variant.+#[test]+fn roundtrip_ordered_aggregate_order_by() -> Result<(), DataFusionError> {+ roundtrip_statement_with_dialect_helper!(+ sql: "SELECT first_name, last_value(age ORDER BY salary) FROM person GROUP BY first_name",+ parser_dialect: GenericDialect {},+ unparser_dialect: UnparserDefaultDialect {},+ expected: @"SELECT person.first_name, last_value(person.age ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",+ );+ roundtrip_statement_with_dialect_helper!(+ sql: "SELECT first_name, first_value(age ORDER BY salary DESC) FROM person GROUP BY first_name",+ parser_dialect: GenericDialect {},+ unparser_dialect: UnparserDefaultDialect {},+ expected: @"SELECT person.first_name, first_value(person.age ORDER BY person.salary DESC NULLS FIRST) FROM person GROUP BY person.first_name",+ );+ roundtrip_statement_with_dialect_helper!(+ sql: "SELECT first_name, array_agg(age ORDER BY salary) FROM person GROUP BY first_name",+ parser_dialect: GenericDialect {},+ unparser_dialect: UnparserDefaultDialect {},+ expected: @"SELECT person.first_name, array_agg(person.age ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",+ );+ roundtrip_statement_with_dialect_helper!(+ sql: "SELECT first_name, string_agg(CAST(age AS VARCHAR), ',' ORDER BY salary) FROM person GROUP BY first_name",+ parser_dialect: GenericDialect {},+ unparser_dialect: UnparserDefaultDialect {},+ expected: @"SELECT person.first_name, string_agg(CAST(person.age AS VARCHAR), ',' ORDER BY person.salary ASC NULLS LAST) FROM person GROUP BY person.first_name",+ );+ roundtrip_statement_with_dialect_helper!(+ sql: "SELECT first_name, array_agg(DISTINCT age ORDER BY salary, id) FROM person GROUP BY first_name",+ parser_dialect: GenericDialect {},+ unparser_dialect: UnparserDefaultDialect {},+ expected: @"SELECT person.first_name, array_agg(DISTINCT person.age ORDER BY person.salary ASC NULLS LAST, person.id ASC NULLS LAST) FROM person GROUP BY person.first_name",+ );+ Ok(())+}+#[test]fn snowflake_flatten_multiple_unnest_cross_join() -> Result<(), DataFusionError> {// Realistic Snowflake pattern: