Apache DataFusion SQL Query Engine
Support DISTINCT ON with aggregation and windows (#22169)
Kumar Ujjawal492e439822eb
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rsindex b7f7d80e70..b0099b8a1d 100644--- a/datafusion/sql/src/select.rs+++ b/datafusion/sql/src/select.rs@@ -25,7 +25,8 @@ use crate::utils::{CheckColumnsMustReferenceAggregatePurpose, CheckColumnsSatisfyExprsPurpose,check_columns_satisfy_exprs, extract_aliases, rebase_expr, resolve_aliases_to_exprs,resolve_columns, resolve_positions_to_exprs, rewrite_recursive_unnest_bottom_up,- rewrite_recursive_unnests_bottom_up,+ rewrite_recursive_unnests_bottom_up, substitute_top_level_alias,+ substitute_top_level_aliases_in_sorts,};use arrow::datatypes::DataType;@@ -69,6 +70,24 @@ struct AggregatePlanResult {qualify_expr: Option<Expr>,/// ORDER BY expressions rewritten to reference aggregate output columnsorder_by_exprs: Vec<SortExpr>,+ /// DISTINCT ON expressions rewritten to reference aggregate output columns+ on_exprs: Vec<Expr>,+}++struct DistinctOnUnnestPlanResult {+ plan: LogicalPlan,+ select_exprs: Vec<Expr>,+ on_exprs: Vec<Expr>,+ order_by_exprs: Vec<SortExpr>,+}++struct RewrittenUnnestExprGroups {+ plan: LogicalPlan,+ expr_groups: Vec<Vec<Expr>>,+}++fn flatten_expr_groups(expr_groups: Vec<Vec<Expr>>) -> Vec<Expr> {+ expr_groups.into_iter().flatten().collect()}impl<S: ContextProvider> SqlToRel<'_, S> {@@ -145,6 +164,36 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// This alias map is resolved and looked up in both having exprs and group by exprslet alias_map = extract_aliases(&select_exprs);+ // DISTINCT ON expressions are parsed alongside HAVING / QUALIFY so+ // they participate in aggregate / window discovery and get rebased+ // through the same pipeline. The SQL nodes are taken out of+ // `select.distinct` so the later match on `Distinct::On` still fires+ // but does not move the original Vec.+ //+ // Resolution precedence matches PostgreSQL and ORDER BY: SELECT+ // aliases win over input columns. For example,+ // SELECT DISTINCT ON (b) a AS b ... GROUP BY a+ // resolves `b` to the alias for `a`, not to a same-named input+ // column.+ let on_exprs_sql: Vec<SQLExpr> = match &mut select.distinct {+ Some(Distinct::On(exprs)) => std::mem::take(exprs),+ _ => Vec::new(),+ };+ let mut on_expr_schema = projected_plan.schema().as_ref().clone();+ on_expr_schema.merge(base_plan.schema());+ let on_exprs_pre_aggr: Vec<Expr> = on_exprs_sql+ .into_iter()+ .map(|e| {+ let expr =+ self.sql_expr_to_logical_expr(e, &on_expr_schema, planner_context)?;+ // PostgreSQL only substitutes an output alias when the whole+ // ON expression is a bare identifier. `b` resolves to the+ // alias; `b + 0` keeps `b` as the input column.+ let expr = substitute_top_level_alias(expr, &alias_map);+ normalize_col(expr, &projected_plan)+ })+ .collect::<Result<Vec<Expr>>>()?;+// Optionally the HAVING expression.let having_expr_opt = select.having@@ -251,12 +300,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// Find aggregates in ORDER BYlet order_by_aggrs = find_aggregate_exprs(order_by_rex.iter().map(|s| &s.expr));- // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY aggregates- // that aren't already in SELECT/HAVING/QUALIFY+ // Find aggregates in DISTINCT ON+ let on_aggrs = find_aggregate_exprs(on_exprs_pre_aggr.iter());++ // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY+ // and DISTINCT ON aggregates that aren't already covered.let mut aggr_exprs = select_having_qualify_aggrs;- for order_by_aggr in order_by_aggrs {- if !aggr_exprs.iter().any(|e| e == &order_by_aggr) {- aggr_exprs.push(order_by_aggr);+ for extra_aggr in order_by_aggrs.into_iter().chain(on_aggrs) {+ if !aggr_exprs.iter().any(|e| e == &extra_aggr) {+ aggr_exprs.push(extra_aggr);}}@@ -267,6 +319,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {having_expr: having_expr_post_aggr,qualify_expr: qualify_expr_post_aggr,order_by_exprs: mut order_by_rex,+ on_exprs: mut on_exprs_post_aggr,} = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {self.aggregate(&base_plan,@@ -274,6 +327,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {having_expr_opt.as_ref(),qualify_expr_opt.as_ref(),&order_by_rex,+ &on_exprs_pre_aggr,&group_by_exprs,&aggr_exprs,)?@@ -290,6 +344,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {having_expr: having_expr_opt,qualify_expr: qualify_expr_opt,order_by_exprs: order_by_rex,+ on_exprs: on_exprs_pre_aggr,},}};@@ -304,12 +359,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// All of the window expressions (deduplicated and rewritten to reference aggregates as// columns from input). Window functions may be sourced from the SELECT list, QUALIFY- // expression, or ORDER BY.+ // expression, ORDER BY, or DISTINCT ON.let window_func_exprs = find_window_exprs(select_exprs_post_aggr.iter().chain(qualify_expr_post_aggr.iter())- .chain(order_by_rex.iter().map(|s| &s.expr)),+ .chain(order_by_rex.iter().map(|s| &s.expr))+ .chain(on_exprs_post_aggr.iter()),);// Process window functions after aggregation as they can reference@@ -336,6 +392,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {}).collect::<Result<Vec<_>>>()?;+ on_exprs_post_aggr = on_exprs_post_aggr+ .iter()+ .map(|expr| rebase_expr(expr, &window_func_exprs, &plan))+ .collect::<Result<Vec<Expr>>>()?;+plan};@@ -377,39 +438,74 @@ impl<S: ContextProvider> SqlToRel<'_, S> {plan};- // Try processing unnest expression or do the final projection- let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;-- // Process distinct clause+ // Process distinct clause. For `DISTINCT ON` combined with+ // aggregation, GROUP BY, or window functions we apply DistinctOn+ // *before* the final projection so grouping columns and ORDER BY+ // tie-breakers that aren't in the user SELECT stay in scope.+ // DistinctOn provides the projection in that case (its select_expr+ // list is wrapped in FIRST_VALUE during lowering).let plan = match select.distinct {- None => Ok(plan),- Some(Distinct::All) => Ok(plan),+ None | Some(Distinct::All) => {+ self.try_process_unnest(plan, select_exprs_post_aggr)?+ }Some(Distinct::Distinct) => {- LogicalPlanBuilder::from(plan).distinct()?.build()+ let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;+ LogicalPlanBuilder::from(plan).distinct()?.build()?}- Some(Distinct::On(on_expr)) => {- if !aggr_exprs.is_empty()- || !group_by_exprs.is_empty()- || !window_func_exprs.is_empty()+ Some(Distinct::On(_)) => {+ if aggr_exprs.is_empty()+ && group_by_exprs.is_empty()+ && window_func_exprs.is_empty(){- return not_impl_err!(- "DISTINCT ON expressions with GROUP BY, aggregation or window functions are not supported "+ // Fast path: no aggregation context. Fuse projection+ // and deduplication into a single DistinctOn over+ // `base_plan`. The sort attached to DistinctOn via+ // `with_sort_expr` later normalizes against base_plan,+ // so a bare ORDER BY alias (e.g. `ORDER BY x` where+ // SELECT has `a AS x`) must be swapped back to the+ // underlying input expression first.+ order_by_rex =+ substitute_top_level_aliases_in_sorts(order_by_rex, &alias_map);+ LogicalPlanBuilder::from(base_plan)+ .distinct_on(on_exprs_post_aggr, select_exprs, None)?+ .build()?+ } else {+ // General path: DistinctOn layered over the post-+ // aggregate / post-window plan (no extra Projection+ // node — DistinctOn's lowering wraps each select_expr+ // in FIRST_VALUE, which acts as the projection).+ //+ // The DistinctOn input has the post-aggregate raw+ // column names (e.g. `max(t.c4)`), not the user-facing+ // SELECT aliases (`agg2`). ORDER BY may reference+ // those aliases — substitute them back to the+ // underlying post-aggregate expression so they+ // resolve against the DistinctOn input.+ let select_alias_map = extract_aliases(&select_exprs_post_aggr);+ order_by_rex = substitute_top_level_aliases_in_sorts(+ order_by_rex,+ &select_alias_map,);- }- let on_expr = on_expr- .into_iter()- .map(|e| {- self.sql_expr_to_logical_expr(e, plan.schema(), planner_context)- })- .collect::<Result<Vec<_>>>()?;+ let DistinctOnUnnestPlanResult {+ plan,+ select_exprs: select_exprs_post_aggr,+ on_exprs: on_exprs_post_aggr,+ order_by_exprs: rewritten_order_by_rex,+ } = self.try_process_distinct_on_unnest(+ plan,+ select_exprs_post_aggr,+ on_exprs_post_aggr,+ order_by_rex,+ )?;+ order_by_rex = rewritten_order_by_rex;- // Build the final plan- LogicalPlanBuilder::from(base_plan)- .distinct_on(on_expr, select_exprs, None)?- .build()+ LogicalPlanBuilder::from(plan)+ .distinct_on(on_exprs_post_aggr, select_exprs_post_aggr, None)?+ .build()?+ }}- }?;+ };// DISTRIBUTE BYlet plan = if !select.distribute_by.is_empty() {@@ -441,98 +537,159 @@ impl<S: ContextProvider> SqlToRel<'_, S> {input: LogicalPlan,select_exprs: Vec<Expr>,) -> Result<LogicalPlan> {+ let RewrittenUnnestExprGroups { plan, expr_groups } = self+ .rewrite_unnest_expr_groups(+ input,+ select_exprs.into_iter().map(|expr| vec![expr]).collect(),+ )?;++ LogicalPlanBuilder::from(plan)+ .project(flatten_expr_groups(expr_groups))?+ .build()+ }++ /// Rewrites SELECT-list UNNESTs while keeping hidden DISTINCT ON / ORDER+ /// BY inputs available to the DistinctOn node.+ fn try_process_distinct_on_unnest(+ &self,+ input: LogicalPlan,+ select_exprs: Vec<Expr>,+ on_exprs: Vec<Expr>,+ order_by_exprs: Vec<SortExpr>,+ ) -> Result<DistinctOnUnnestPlanResult> {+ let select_len = select_exprs.len();+ let on_len = on_exprs.len();+ let mut expr_groups = select_exprs+ .into_iter()+ .map(|expr| vec![expr])+ .collect::<Vec<_>>();+ expr_groups.extend(on_exprs.into_iter().map(|expr| vec![expr]));+ expr_groups.extend(+ order_by_exprs+ .iter()+ .map(|sort_expr| vec![sort_expr.expr.clone()]),+ );++ let RewrittenUnnestExprGroups {+ plan,+ mut expr_groups,+ } = self.rewrite_unnest_expr_groups(input, expr_groups)?;++ let rewritten_select_exprs =+ flatten_expr_groups(expr_groups.drain(..select_len).collect());+ let rewritten_on_exprs = expr_groups+ .drain(..on_len)+ .map(|exprs| self.expect_single_distinct_on_expr(exprs, "DISTINCT ON"))+ .collect::<Result<Vec<_>>>()?;+ let rewritten_order_by_exprs = order_by_exprs+ .into_iter()+ .zip(expr_groups)+ .map(|(sort_expr, exprs)| {+ Ok(sort_expr+ .with_expr(self.expect_single_distinct_on_expr(exprs, "ORDER BY")?))+ })+ .collect::<Result<Vec<_>>>()?;++ Ok(DistinctOnUnnestPlanResult {+ plan,+ select_exprs: rewritten_select_exprs,+ on_exprs: rewritten_on_exprs,+ order_by_exprs: rewritten_order_by_exprs,+ })+ }++ fn expect_single_distinct_on_expr(+ &self,+ exprs: Vec<Expr>,+ clause: &str,+ ) -> Result<Expr> {+ if exprs.len() == 1 {+ return Ok(exprs.into_iter().next().expect("len checked above"));+ }++ not_impl_err!(+ "{clause} expressions that expand to multiple columns are not supported with DISTINCT ON"+ )+ }++ fn rewrite_unnest_expr_groups(+ &self,+ input: LogicalPlan,+ expr_groups: Vec<Vec<Expr>>,+ ) -> Result<RewrittenUnnestExprGroups> {// Try process group by unnestlet input = self.try_process_aggregate_unnest(input)?;let mut intermediate_plan = input;- let mut intermediate_select_exprs = select_exprs;- // Fast path: If there is are no unnests in the select_exprs, wrap the plan in a projection- if !intermediate_select_exprs- .iter()- .any(has_unnest_expr_recursively)- {- return LogicalPlanBuilder::from(intermediate_plan)- .project(intermediate_select_exprs)?- .build();- }+ let mut intermediate_expr_groups = expr_groups;- // Each expr in select_exprs can contains multiple unnest stage- // The transformation happen bottom up, one at a time for each iteration- // Only exhaust the loop if no more unnest transformation is found- for i in 0.. {+ loop {let mut unnest_columns = IndexMap::new();- // from which column used for projection, before the unnest happen- // including non unnest column and unnest column+ // from which columns used for projection, before the unnest happen+ // including non unnest columns and unnest columnslet mut inner_projection_exprs = vec![];+ let mut outer_expr_groups =+ Vec::with_capacity(intermediate_expr_groups.len());++ for expr_group in &intermediate_expr_groups {+ let mut outer_expr_group = vec![];+ for expr in expr_group {+ let mut rewritten_exprs = rewrite_recursive_unnest_bottom_up(+ &intermediate_plan,+ &mut unnest_columns,+ &mut inner_projection_exprs,+ expr,+ )?;- // expr returned here maybe different from the originals in inner_projection_exprs- // for example:- // - unnest(struct_col) will be transformed into struct_col.field1, struct_col.field2- // - unnest(array_col) will be transformed into array_col.element- // - unnest(array_col) + 1 will be transformed into array_col.element +1- let mut outer_projection_exprs = vec![];- for expr in &intermediate_select_exprs {- let mut rewritten_exprs = rewrite_recursive_unnest_bottom_up(- &intermediate_plan,- &mut unnest_columns,- &mut inner_projection_exprs,- expr,- )?;+ if let Some(columns) =+ self.get_struct_unnest_columns(&intermediate_plan, expr)?+ {+ rewritten_exprs = rewritten_exprs+ .into_iter()+ .zip(columns)+ .map(|(expr, column)| expr.alias(column.flat_name()))+ .collect();+ }- if let Some(columns) =- self.get_struct_unnest_columns(&intermediate_plan, expr)?- {- rewritten_exprs = rewritten_exprs- .into_iter()- .zip(columns)- .map(|(expr, column)| expr.alias(column.flat_name()))- .collect();+ outer_expr_group.extend(rewritten_exprs);}-- outer_projection_exprs.extend(rewritten_exprs);+ outer_expr_groups.push(outer_expr_group);}// No more unnest is possibleif unnest_columns.is_empty() {- // The original expr does not contain any unnest- if i == 0 {- return LogicalPlanBuilder::from(intermediate_plan)- .project(intermediate_select_exprs)?- .build();- }- break;- } else {- // Set preserve_nulls to false to ensure compatibility with DuckDB and PostgreSQL- let mut unnest_options = UnnestOptions::new().with_preserve_nulls(false);- let mut unnest_col_vec = vec![];+ return Ok(RewrittenUnnestExprGroups {+ plan: intermediate_plan,+ expr_groups: intermediate_expr_groups,+ });+ }- for (col, maybe_list_unnest) in unnest_columns.into_iter() {- if let Some(list_unnest) = maybe_list_unnest {- unnest_options = list_unnest.into_iter().fold(- unnest_options,- |options, unnest_list| {- options.with_recursions(RecursionUnnestOption {- input_column: col.clone(),- output_column: unnest_list.output_column,- depth: unnest_list.depth,- })- },- );- }- unnest_col_vec.push(col);+ // Set preserve_nulls to false to ensure compatibility with DuckDB and PostgreSQL+ let mut unnest_options = UnnestOptions::new().with_preserve_nulls(false);+ let mut unnest_col_vec = vec![];++ for (col, maybe_list_unnest) in unnest_columns.into_iter() {+ if let Some(list_unnest) = maybe_list_unnest {+ unnest_options = list_unnest.into_iter().fold(+ unnest_options,+ |options, unnest_list| {+ options.with_recursions(RecursionUnnestOption {+ input_column: col.clone(),+ output_column: unnest_list.output_column,+ depth: unnest_list.depth,+ })+ },+ );}- let plan = LogicalPlanBuilder::from(intermediate_plan)- .project(inner_projection_exprs)?- .unnest_columns_with_options(unnest_col_vec, unnest_options)?- .build()?;- intermediate_plan = plan;- intermediate_select_exprs = outer_projection_exprs;+ unnest_col_vec.push(col);}- }- LogicalPlanBuilder::from(intermediate_plan)- .project(intermediate_select_exprs)?- .build()+ intermediate_plan = LogicalPlanBuilder::from(intermediate_plan)+ .project(inner_projection_exprs)?+ .unnest_columns_with_options(unnest_col_vec, unnest_options)?+ .build()?;+ intermediate_expr_groups = outer_expr_groups;+ }}fn get_struct_unnest_columns(@@ -1017,6 +1174,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {having_expr_opt: Option<&Expr>,qualify_expr_opt: Option<&Expr>,order_by_exprs: &[SortExpr],+ on_exprs: &[Expr],group_by_exprs: &[Expr],aggr_exprs: &[Expr],) -> Result<AggregatePlanResult> {@@ -1183,12 +1341,28 @@ impl<S: ContextProvider> SqlToRel<'_, S> {),)?;+ // Rewrite the DISTINCT ON expressions to use the columns produced by+ // the aggregation. Same shape as ORDER BY rewriting so a hidden+ // grouping column or a raw aggregate expression in ON is resolved.+ let on_exprs_post_aggr = on_exprs+ .iter()+ .map(|expr| rebase_expr(expr, &aggr_projection_exprs, input))+ .collect::<Result<Vec<Expr>>>()?;+ check_columns_satisfy_exprs(+ &all_valid_exprs,+ &on_exprs_post_aggr,+ CheckColumnsSatisfyExprsPurpose::Aggregate(+ CheckColumnsMustReferenceAggregatePurpose::DistinctOn,+ ),+ )?;+Ok(AggregatePlanResult {plan,select_exprs: select_exprs_post_aggr,having_expr: having_expr_post_aggr,qualify_expr: qualify_expr_post_aggr,order_by_exprs: order_by_post_aggr,+ on_exprs: on_exprs_post_aggr,})}diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rsindex 1a76dd69f4..3b571eed27 100644--- a/datafusion/sql/src/utils.rs+++ b/datafusion/sql/src/utils.rs@@ -35,7 +35,7 @@ use datafusion_expr::expr::{};use datafusion_expr::utils::{expr_as_column_expr, find_column_exprs};use datafusion_expr::{- ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, col, expr_vec_fmt,+ ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, SortExpr, col, expr_vec_fmt,};use indexmap::IndexMap;@@ -98,6 +98,7 @@ pub(crate) enum CheckColumnsMustReferenceAggregatePurpose {Having,Qualify,OrderBy,+ DistinctOn,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -120,6 +121,9 @@ impl CheckColumnsSatisfyExprsPurpose {Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::OrderBy) => {"Column in ORDER BY must be in GROUP BY or an aggregate function"}+ Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::DistinctOn) => {+ "Column in DISTINCT ON must be in GROUP BY or an aggregate function"+ }}}@@ -202,6 +206,42 @@ pub(crate) fn extract_aliases(exprs: &[Expr]) -> HashMap<String, Expr> {.collect::<HashMap<String, Expr>>()}+/// If `expr` is a bare unqualified `Column` whose name matches a SELECT+/// alias, swap it for the alias's underlying expression. Nested occurrences+/// are left alone: PostgreSQL only resolves a top-level identifier as an+/// output alias in clauses like ORDER BY and DISTINCT ON.+pub(crate) fn substitute_top_level_alias(+ expr: Expr,+ aliases: &HashMap<String, Expr>,+) -> Expr {+ if let Expr::Column(col) = &expr+ && col.relation.is_none()+ && let Some(underlying) = aliases.get(&col.name)+ {+ return underlying.clone();+ }++ expr+}++/// Applies [`substitute_top_level_alias`] to each sort expression.+pub(crate) fn substitute_top_level_aliases_in_sorts(+ sort_exprs: Vec<SortExpr>,+ aliases: &HashMap<String, Expr>,+) -> Vec<SortExpr> {+ if aliases.is_empty() {+ return sort_exprs;+ }++ sort_exprs+ .into_iter()+ .map(|sort_expr| {+ sort_expr+ .with_expr(substitute_top_level_alias(sort_expr.expr.clone(), aliases))+ })+ .collect()+}+/// Given an expression that's literal int encoding position, lookup the corresponding expression/// in the select_exprs list, if the index is within the bounds and it is indeed a position literal,/// otherwise, returns planning error.diff --git a/datafusion/sqllogictest/test_files/distinct_on.slt b/datafusion/sqllogictest/test_files/distinct_on.sltindex 5b18915080..0659b9c208 100644--- a/datafusion/sqllogictest/test_files/distinct_on.slt+++ b/datafusion/sqllogictest/test_files/distinct_on.slt@@ -195,3 +195,233 @@ RESET datafusion.explain.logical_plan_only;statement okdrop table t;++# DISTINCT ON combined with GROUP BY + aggregation (issue #17256).+# ON references a grouping column; ORDER BY uses an aggregate alias.+query TII+SELECT DISTINCT ON (c1) c1, c3, max(c4) AS agg2+FROM aggregate_test_100 GROUP BY c1, c3 ORDER BY c1, agg2;+----+a 65 -28462+b -60 -21739+c 3 -30508+d 102 -24558+e -56 -31500++# DISTINCT ON referencing a SELECT alias for an aggregate.+query TI+SELECT DISTINCT ON (agg2) c1, max(c4) AS agg2+FROM aggregate_test_100 GROUP BY c1 ORDER BY agg2;+----+b 25286+c 29106+d 31106+a 32064+e 32514++# DISTINCT ON with a scalar function over a grouping column.+query TI+SELECT DISTINCT ON (upper(c1)) c1, sum(c3) FROM aggregate_test_100+GROUP BY c1 ORDER BY upper(c1);+----+a -385+b -111+c -28+d 458+e 847++# Hidden ORDER BY tie-breaker: c3 is in GROUP BY and ORDER BY but+# NOT in the SELECT list. PostgreSQL accepts this.+query TI+SELECT DISTINCT ON (c1) c1, max(c4) AS agg2+FROM aggregate_test_100 GROUP BY c1, c3 ORDER BY c1, c3;+----+a 11640+b 19316+c -30187+d 5613+e 13611++# Hidden DISTINCT ON key: ON references a grouping column that is+# NOT in the SELECT list. ORDER BY adds a deterministic tie-breaker.+query II+SELECT DISTINCT ON (c1) c2 % 2, count(*) AS n+FROM aggregate_test_100 GROUP BY c1, c2 % 2 ORDER BY c1, c2 % 2;+----+0 7+0 9+0 11+0 6+0 12++# Raw aggregate expression in DISTINCT ON (not via an alias).+query TI+SELECT DISTINCT ON (sum(c3)) c1, sum(c3) AS total+FROM aggregate_test_100 GROUP BY c1 ORDER BY sum(c3);+----+a -385+b -111+c -28+d 458+e 847++# DISTINCT ON with HAVING.+query TI+SELECT DISTINCT ON (c1) c1, count(*) AS cnt FROM aggregate_test_100+GROUP BY c1 HAVING count(*) > 10 ORDER BY c1, cnt DESC;+----+a 21+b 19+c 21+d 18+e 21++# DISTINCT ON combined with a window function over a unique ordering+# key, so the test is fully deterministic.+query II+WITH t(id, v) AS (VALUES (1, 10), (2, 20), (3, 10), (4, 30), (5, 20))+SELECT DISTINCT ON (v) v, row_number() OVER (ORDER BY id) AS rn+FROM t ORDER BY v, rn;+----+10 1+20 2+30 4++# Raw window expression in DISTINCT ON (not via an alias). Uses a+# unique ordering key so row_number is deterministic.+query II+WITH t(id, v) AS (VALUES (1, 10), (2, 20), (3, 30), (4, 40))+SELECT DISTINCT ON (row_number() OVER (ORDER BY id)) id, v+FROM t+ORDER BY row_number() OVER (ORDER BY id);+----+1 10+2 20+3 30+4 40++# Qualified join columns with potential alias conflict.+query TII+WITH t1(k, v) AS (VALUES ('x', 1), ('x', 2), ('y', 3)),+ t2(k, w) AS (VALUES ('x', 10), ('y', 20))+SELECT DISTINCT ON (t1.k) t1.k, sum(t1.v) AS s, max(t2.w) AS mw+FROM t1 JOIN t2 ON t1.k = t2.k+GROUP BY t1.k ORDER BY t1.k;+----+x 3 10+y 3 20++# DISTINCT ON name conflicts with an input column of the same name.+# PostgreSQL resolves `b` to the SELECT alias `a AS b`, not the input+# column `t.b`. Groups should be keyed by `a`, not `t.b`.+query TI+WITH t(a, b) AS (VALUES ('x', 1), ('x', 2), ('y', 1))+SELECT DISTINCT ON (b) a AS b, count(*) AS n+FROM t GROUP BY a ORDER BY b;+----+x 2+y 1++# A bare alias resolves to the SELECT expression, but inside a larger+# expression the same identifier refers to the input column. Postgres:+# ORDER BY a, b + 0 DESC+# uses `a` (post-aggregate) and `t.b + 0`, so for a=100 the row with+# t.b=2 wins (sum=2). DataFusion must not recursively swap `b` inside+# `b + 0` for the alias.+query II+WITH t(a, b) AS (VALUES (100, 1), (100, 2), (200, 1))+SELECT DISTINCT ON (a) a AS b, sum(b) AS s+FROM t GROUP BY a, b ORDER BY a, b + 0 DESC;+----+100 2+200 1++# A nested ORDER BY expression over a SELECT alias is still rejected in+# the post-aggregate DISTINCT ON path.+query error No field named agg2+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))+SELECT DISTINCT ON (a) a, max(b) AS agg2+FROM t GROUP BY a ORDER BY a, agg2 + 1;++# DISTINCT ON after aggregation still needs the SELECT-list unnest+# rewrite, and bare ORDER BY aliases should keep working against the+# rewritten DistinctOn input.+query II+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))+SELECT DISTINCT ON (a) a, unnest(array_agg(b)) AS b+FROM t GROUP BY a ORDER BY a, b DESC;+----+1 20+2 30++# DISTINCT ON after aggregation also needs to keep multi-column SELECT+# expansions from struct unnest working.+query III+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))+SELECT DISTINCT ON (a) a, unnest(struct(max(b), min(b)))+FROM t GROUP BY a ORDER BY a;+----+1 20 10+2 30 30++# DISTINCT ON keys that expand to multiple columns are rejected.+query error DISTINCT ON expressions that expand to multiple columns are not supported with DISTINCT ON+WITH t(a, b) AS (VALUES (1, 10), (2, 20))+SELECT DISTINCT ON (unnest(struct(max(b), min(b)))) a+FROM t GROUP BY a ORDER BY unnest(struct(max(b), min(b))), a;++# ORDER BY tie-breakers that expand to multiple columns are rejected.+query error ORDER BY expressions that expand to multiple columns are not supported with DISTINCT ON+WITH t(a, b) AS (VALUES (1, 10), (2, 20))+SELECT DISTINCT ON (a) a, max(b)+FROM t GROUP BY a ORDER BY a, unnest(struct(max(b), min(b)));++# Fast path (no aggregation): a bare ORDER BY alias must resolve to+# its underlying SELECT expression so that the sort attached to+# DistinctOn normalizes against the base plan.+query T+WITH t(a, b) AS (VALUES ('x', 1), ('x', 2), ('y', 3))+SELECT DISTINCT ON (x) a AS x FROM t ORDER BY x;+----+x+y++# EXPLAIN for the post-aggregation case.+statement ok+set datafusion.explain.logical_plan_only = true;++query TT+explain SELECT DISTINCT ON (c1) c1, max(c4) AS agg2+FROM aggregate_test_100 GROUP BY c1 ORDER BY c1, agg2;+----+logical_plan+01)Projection: first_value(aggregate_test_100.c1) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST] AS c1, first_value(agg2) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST] AS agg2+02)--Sort: aggregate_test_100.c1 ASC NULLS LAST+03)----Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[first_value(aggregate_test_100.c1) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST], first_value(max(aggregate_test_100.c4) AS agg2) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST]]]+04)------Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[max(aggregate_test_100.c4)]]+05)--------TableScan: aggregate_test_100 projection=[c1, c4]++statement ok+RESET datafusion.explain.logical_plan_only;++# Ordinal ORDER BY still works in the post-aggregate DISTINCT ON path.+query TI+SELECT DISTINCT ON (c1) c1, max(c4)+FROM aggregate_test_100 GROUP BY c1 ORDER BY 1, 2;+----+a 32064+b 25286+c 29106+d 31106+e 32514++# Synthetic repro for issue #17256.+query TIR+WITH t(a, b, c) AS (+ VALUES ('x', 1, 10.0), ('x', 1, 20.0), ('y', 2, 30.0)+)+SELECT DISTINCT ON (a) a, b, sum(c) AS total+FROM t GROUP BY a, b ORDER BY a, total DESC;+----+x 1 30+y 2 30