Skip to content

Apache DataFusion SQL Query Engine

Support DISTINCT ON with aggregation and windows (#22169)

Kumar Ujjawal492e439822eb
diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs
index b7f7d80e70..b0099b8a1d 100644
--- a/datafusion/sql/src/select.rs
+++ b/datafusion/sql/src/select.rs
@@ -25,7 +25,8 @@ use crate::utils::{
CheckColumnsMustReferenceAggregatePurpose, CheckColumnsSatisfyExprsPurpose,
check_columns_satisfy_exprs, extract_aliases, rebase_expr, resolve_aliases_to_exprs,
resolve_columns, resolve_positions_to_exprs, rewrite_recursive_unnest_bottom_up,
- rewrite_recursive_unnests_bottom_up,
+ rewrite_recursive_unnests_bottom_up, substitute_top_level_alias,
+ substitute_top_level_aliases_in_sorts,
};
use arrow::datatypes::DataType;
@@ -69,6 +70,24 @@ struct AggregatePlanResult {
qualify_expr: Option<Expr>,
/// ORDER BY expressions rewritten to reference aggregate output columns
order_by_exprs: Vec<SortExpr>,
+ /// DISTINCT ON expressions rewritten to reference aggregate output columns
+ on_exprs: Vec<Expr>,
+}
+
+struct DistinctOnUnnestPlanResult {
+ plan: LogicalPlan,
+ select_exprs: Vec<Expr>,
+ on_exprs: Vec<Expr>,
+ order_by_exprs: Vec<SortExpr>,
+}
+
+struct RewrittenUnnestExprGroups {
+ plan: LogicalPlan,
+ expr_groups: Vec<Vec<Expr>>,
+}
+
+fn flatten_expr_groups(expr_groups: Vec<Vec<Expr>>) -> Vec<Expr> {
+ expr_groups.into_iter().flatten().collect()
}
impl<S: ContextProvider> SqlToRel<'_, S> {
@@ -145,6 +164,36 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// This alias map is resolved and looked up in both having exprs and group by exprs
let alias_map = extract_aliases(&select_exprs);
+ // DISTINCT ON expressions are parsed alongside HAVING / QUALIFY so
+ // they participate in aggregate / window discovery and get rebased
+ // through the same pipeline. The SQL nodes are taken out of
+ // `select.distinct` so the later match on `Distinct::On` still fires
+ // but does not move the original Vec.
+ //
+ // Resolution precedence matches PostgreSQL and ORDER BY: SELECT
+ // aliases win over input columns. For example,
+ // SELECT DISTINCT ON (b) a AS b ... GROUP BY a
+ // resolves `b` to the alias for `a`, not to a same-named input
+ // column.
+ let on_exprs_sql: Vec<SQLExpr> = match &mut select.distinct {
+ Some(Distinct::On(exprs)) => std::mem::take(exprs),
+ _ => Vec::new(),
+ };
+ let mut on_expr_schema = projected_plan.schema().as_ref().clone();
+ on_expr_schema.merge(base_plan.schema());
+ let on_exprs_pre_aggr: Vec<Expr> = on_exprs_sql
+ .into_iter()
+ .map(|e| {
+ let expr =
+ self.sql_expr_to_logical_expr(e, &on_expr_schema, planner_context)?;
+ // PostgreSQL only substitutes an output alias when the whole
+ // ON expression is a bare identifier. `b` resolves to the
+ // alias; `b + 0` keeps `b` as the input column.
+ let expr = substitute_top_level_alias(expr, &alias_map);
+ normalize_col(expr, &projected_plan)
+ })
+ .collect::<Result<Vec<Expr>>>()?;
+
// Optionally the HAVING expression.
let having_expr_opt = select
.having
@@ -251,12 +300,15 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// Find aggregates in ORDER BY
let order_by_aggrs = find_aggregate_exprs(order_by_rex.iter().map(|s| &s.expr));
- // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY aggregates
- // that aren't already in SELECT/HAVING/QUALIFY
+ // Find aggregates in DISTINCT ON
+ let on_aggrs = find_aggregate_exprs(on_exprs_pre_aggr.iter());
+
+ // Combine: all aggregates from SELECT/HAVING/QUALIFY, plus ORDER BY
+ // and DISTINCT ON aggregates that aren't already covered.
let mut aggr_exprs = select_having_qualify_aggrs;
- for order_by_aggr in order_by_aggrs {
- if !aggr_exprs.iter().any(|e| e == &order_by_aggr) {
- aggr_exprs.push(order_by_aggr);
+ for extra_aggr in order_by_aggrs.into_iter().chain(on_aggrs) {
+ if !aggr_exprs.iter().any(|e| e == &extra_aggr) {
+ aggr_exprs.push(extra_aggr);
}
}
@@ -267,6 +319,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
having_expr: having_expr_post_aggr,
qualify_expr: qualify_expr_post_aggr,
order_by_exprs: mut order_by_rex,
+ on_exprs: mut on_exprs_post_aggr,
} = if !group_by_exprs.is_empty() || !aggr_exprs.is_empty() {
self.aggregate(
&base_plan,
@@ -274,6 +327,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
having_expr_opt.as_ref(),
qualify_expr_opt.as_ref(),
&order_by_rex,
+ &on_exprs_pre_aggr,
&group_by_exprs,
&aggr_exprs,
)?
@@ -290,6 +344,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
having_expr: having_expr_opt,
qualify_expr: qualify_expr_opt,
order_by_exprs: order_by_rex,
+ on_exprs: on_exprs_pre_aggr,
},
}
};
@@ -304,12 +359,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// All of the window expressions (deduplicated and rewritten to reference aggregates as
// columns from input). Window functions may be sourced from the SELECT list, QUALIFY
- // expression, or ORDER BY.
+ // expression, ORDER BY, or DISTINCT ON.
let window_func_exprs = find_window_exprs(
select_exprs_post_aggr
.iter()
.chain(qualify_expr_post_aggr.iter())
- .chain(order_by_rex.iter().map(|s| &s.expr)),
+ .chain(order_by_rex.iter().map(|s| &s.expr))
+ .chain(on_exprs_post_aggr.iter()),
);
// Process window functions after aggregation as they can reference
@@ -336,6 +392,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
})
.collect::<Result<Vec<_>>>()?;
+ on_exprs_post_aggr = on_exprs_post_aggr
+ .iter()
+ .map(|expr| rebase_expr(expr, &window_func_exprs, &plan))
+ .collect::<Result<Vec<Expr>>>()?;
+
plan
};
@@ -377,39 +438,74 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
plan
};
- // Try processing unnest expression or do the final projection
- let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;
-
- // Process distinct clause
+ // Process distinct clause. For `DISTINCT ON` combined with
+ // aggregation, GROUP BY, or window functions we apply DistinctOn
+ // *before* the final projection so grouping columns and ORDER BY
+ // tie-breakers that aren't in the user SELECT stay in scope.
+ // DistinctOn provides the projection in that case (its select_expr
+ // list is wrapped in FIRST_VALUE during lowering).
let plan = match select.distinct {
- None => Ok(plan),
- Some(Distinct::All) => Ok(plan),
+ None | Some(Distinct::All) => {
+ self.try_process_unnest(plan, select_exprs_post_aggr)?
+ }
Some(Distinct::Distinct) => {
- LogicalPlanBuilder::from(plan).distinct()?.build()
+ let plan = self.try_process_unnest(plan, select_exprs_post_aggr)?;
+ LogicalPlanBuilder::from(plan).distinct()?.build()?
}
- Some(Distinct::On(on_expr)) => {
- if !aggr_exprs.is_empty()
- || !group_by_exprs.is_empty()
- || !window_func_exprs.is_empty()
+ Some(Distinct::On(_)) => {
+ if aggr_exprs.is_empty()
+ && group_by_exprs.is_empty()
+ && window_func_exprs.is_empty()
{
- return not_impl_err!(
- "DISTINCT ON expressions with GROUP BY, aggregation or window functions are not supported "
+ // Fast path: no aggregation context. Fuse projection
+ // and deduplication into a single DistinctOn over
+ // `base_plan`. The sort attached to DistinctOn via
+ // `with_sort_expr` later normalizes against base_plan,
+ // so a bare ORDER BY alias (e.g. `ORDER BY x` where
+ // SELECT has `a AS x`) must be swapped back to the
+ // underlying input expression first.
+ order_by_rex =
+ substitute_top_level_aliases_in_sorts(order_by_rex, &alias_map);
+ LogicalPlanBuilder::from(base_plan)
+ .distinct_on(on_exprs_post_aggr, select_exprs, None)?
+ .build()?
+ } else {
+ // General path: DistinctOn layered over the post-
+ // aggregate / post-window plan (no extra Projection
+ // node — DistinctOn's lowering wraps each select_expr
+ // in FIRST_VALUE, which acts as the projection).
+ //
+ // The DistinctOn input has the post-aggregate raw
+ // column names (e.g. `max(t.c4)`), not the user-facing
+ // SELECT aliases (`agg2`). ORDER BY may reference
+ // those aliases — substitute them back to the
+ // underlying post-aggregate expression so they
+ // resolve against the DistinctOn input.
+ let select_alias_map = extract_aliases(&select_exprs_post_aggr);
+ order_by_rex = substitute_top_level_aliases_in_sorts(
+ order_by_rex,
+ &select_alias_map,
);
- }
- let on_expr = on_expr
- .into_iter()
- .map(|e| {
- self.sql_expr_to_logical_expr(e, plan.schema(), planner_context)
- })
- .collect::<Result<Vec<_>>>()?;
+ let DistinctOnUnnestPlanResult {
+ plan,
+ select_exprs: select_exprs_post_aggr,
+ on_exprs: on_exprs_post_aggr,
+ order_by_exprs: rewritten_order_by_rex,
+ } = self.try_process_distinct_on_unnest(
+ plan,
+ select_exprs_post_aggr,
+ on_exprs_post_aggr,
+ order_by_rex,
+ )?;
+ order_by_rex = rewritten_order_by_rex;
- // Build the final plan
- LogicalPlanBuilder::from(base_plan)
- .distinct_on(on_expr, select_exprs, None)?
- .build()
+ LogicalPlanBuilder::from(plan)
+ .distinct_on(on_exprs_post_aggr, select_exprs_post_aggr, None)?
+ .build()?
+ }
}
- }?;
+ };
// DISTRIBUTE BY
let plan = if !select.distribute_by.is_empty() {
@@ -441,98 +537,159 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
input: LogicalPlan,
select_exprs: Vec<Expr>,
) -> Result<LogicalPlan> {
+ let RewrittenUnnestExprGroups { plan, expr_groups } = self
+ .rewrite_unnest_expr_groups(
+ input,
+ select_exprs.into_iter().map(|expr| vec![expr]).collect(),
+ )?;
+
+ LogicalPlanBuilder::from(plan)
+ .project(flatten_expr_groups(expr_groups))?
+ .build()
+ }
+
+ /// Rewrites SELECT-list UNNESTs while keeping hidden DISTINCT ON / ORDER
+ /// BY inputs available to the DistinctOn node.
+ fn try_process_distinct_on_unnest(
+ &self,
+ input: LogicalPlan,
+ select_exprs: Vec<Expr>,
+ on_exprs: Vec<Expr>,
+ order_by_exprs: Vec<SortExpr>,
+ ) -> Result<DistinctOnUnnestPlanResult> {
+ let select_len = select_exprs.len();
+ let on_len = on_exprs.len();
+ let mut expr_groups = select_exprs
+ .into_iter()
+ .map(|expr| vec![expr])
+ .collect::<Vec<_>>();
+ expr_groups.extend(on_exprs.into_iter().map(|expr| vec![expr]));
+ expr_groups.extend(
+ order_by_exprs
+ .iter()
+ .map(|sort_expr| vec![sort_expr.expr.clone()]),
+ );
+
+ let RewrittenUnnestExprGroups {
+ plan,
+ mut expr_groups,
+ } = self.rewrite_unnest_expr_groups(input, expr_groups)?;
+
+ let rewritten_select_exprs =
+ flatten_expr_groups(expr_groups.drain(..select_len).collect());
+ let rewritten_on_exprs = expr_groups
+ .drain(..on_len)
+ .map(|exprs| self.expect_single_distinct_on_expr(exprs, "DISTINCT ON"))
+ .collect::<Result<Vec<_>>>()?;
+ let rewritten_order_by_exprs = order_by_exprs
+ .into_iter()
+ .zip(expr_groups)
+ .map(|(sort_expr, exprs)| {
+ Ok(sort_expr
+ .with_expr(self.expect_single_distinct_on_expr(exprs, "ORDER BY")?))
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(DistinctOnUnnestPlanResult {
+ plan,
+ select_exprs: rewritten_select_exprs,
+ on_exprs: rewritten_on_exprs,
+ order_by_exprs: rewritten_order_by_exprs,
+ })
+ }
+
+ fn expect_single_distinct_on_expr(
+ &self,
+ exprs: Vec<Expr>,
+ clause: &str,
+ ) -> Result<Expr> {
+ if exprs.len() == 1 {
+ return Ok(exprs.into_iter().next().expect("len checked above"));
+ }
+
+ not_impl_err!(
+ "{clause} expressions that expand to multiple columns are not supported with DISTINCT ON"
+ )
+ }
+
+ fn rewrite_unnest_expr_groups(
+ &self,
+ input: LogicalPlan,
+ expr_groups: Vec<Vec<Expr>>,
+ ) -> Result<RewrittenUnnestExprGroups> {
// Try process group by unnest
let input = self.try_process_aggregate_unnest(input)?;
let mut intermediate_plan = input;
- let mut intermediate_select_exprs = select_exprs;
- // Fast path: If there is are no unnests in the select_exprs, wrap the plan in a projection
- if !intermediate_select_exprs
- .iter()
- .any(has_unnest_expr_recursively)
- {
- return LogicalPlanBuilder::from(intermediate_plan)
- .project(intermediate_select_exprs)?
- .build();
- }
+ let mut intermediate_expr_groups = expr_groups;
- // Each expr in select_exprs can contains multiple unnest stage
- // The transformation happen bottom up, one at a time for each iteration
- // Only exhaust the loop if no more unnest transformation is found
- for i in 0.. {
+ loop {
let mut unnest_columns = IndexMap::new();
- // from which column used for projection, before the unnest happen
- // including non unnest column and unnest column
+ // from which columns used for projection, before the unnest happen
+ // including non unnest columns and unnest columns
let mut inner_projection_exprs = vec![];
+ let mut outer_expr_groups =
+ Vec::with_capacity(intermediate_expr_groups.len());
+
+ for expr_group in &intermediate_expr_groups {
+ let mut outer_expr_group = vec![];
+ for expr in expr_group {
+ let mut rewritten_exprs = rewrite_recursive_unnest_bottom_up(
+ &intermediate_plan,
+ &mut unnest_columns,
+ &mut inner_projection_exprs,
+ expr,
+ )?;
- // expr returned here maybe different from the originals in inner_projection_exprs
- // for example:
- // - unnest(struct_col) will be transformed into struct_col.field1, struct_col.field2
- // - unnest(array_col) will be transformed into array_col.element
- // - unnest(array_col) + 1 will be transformed into array_col.element +1
- let mut outer_projection_exprs = vec![];
- for expr in &intermediate_select_exprs {
- let mut rewritten_exprs = rewrite_recursive_unnest_bottom_up(
- &intermediate_plan,
- &mut unnest_columns,
- &mut inner_projection_exprs,
- expr,
- )?;
+ if let Some(columns) =
+ self.get_struct_unnest_columns(&intermediate_plan, expr)?
+ {
+ rewritten_exprs = rewritten_exprs
+ .into_iter()
+ .zip(columns)
+ .map(|(expr, column)| expr.alias(column.flat_name()))
+ .collect();
+ }
- if let Some(columns) =
- self.get_struct_unnest_columns(&intermediate_plan, expr)?
- {
- rewritten_exprs = rewritten_exprs
- .into_iter()
- .zip(columns)
- .map(|(expr, column)| expr.alias(column.flat_name()))
- .collect();
+ outer_expr_group.extend(rewritten_exprs);
}
-
- outer_projection_exprs.extend(rewritten_exprs);
+ outer_expr_groups.push(outer_expr_group);
}
// No more unnest is possible
if unnest_columns.is_empty() {
- // The original expr does not contain any unnest
- if i == 0 {
- return LogicalPlanBuilder::from(intermediate_plan)
- .project(intermediate_select_exprs)?
- .build();
- }
- break;
- } else {
- // Set preserve_nulls to false to ensure compatibility with DuckDB and PostgreSQL
- let mut unnest_options = UnnestOptions::new().with_preserve_nulls(false);
- let mut unnest_col_vec = vec![];
+ return Ok(RewrittenUnnestExprGroups {
+ plan: intermediate_plan,
+ expr_groups: intermediate_expr_groups,
+ });
+ }
- for (col, maybe_list_unnest) in unnest_columns.into_iter() {
- if let Some(list_unnest) = maybe_list_unnest {
- unnest_options = list_unnest.into_iter().fold(
- unnest_options,
- |options, unnest_list| {
- options.with_recursions(RecursionUnnestOption {
- input_column: col.clone(),
- output_column: unnest_list.output_column,
- depth: unnest_list.depth,
- })
- },
- );
- }
- unnest_col_vec.push(col);
+ // Set preserve_nulls to false to ensure compatibility with DuckDB and PostgreSQL
+ let mut unnest_options = UnnestOptions::new().with_preserve_nulls(false);
+ let mut unnest_col_vec = vec![];
+
+ for (col, maybe_list_unnest) in unnest_columns.into_iter() {
+ if let Some(list_unnest) = maybe_list_unnest {
+ unnest_options = list_unnest.into_iter().fold(
+ unnest_options,
+ |options, unnest_list| {
+ options.with_recursions(RecursionUnnestOption {
+ input_column: col.clone(),
+ output_column: unnest_list.output_column,
+ depth: unnest_list.depth,
+ })
+ },
+ );
}
- let plan = LogicalPlanBuilder::from(intermediate_plan)
- .project(inner_projection_exprs)?
- .unnest_columns_with_options(unnest_col_vec, unnest_options)?
- .build()?;
- intermediate_plan = plan;
- intermediate_select_exprs = outer_projection_exprs;
+ unnest_col_vec.push(col);
}
- }
- LogicalPlanBuilder::from(intermediate_plan)
- .project(intermediate_select_exprs)?
- .build()
+ intermediate_plan = LogicalPlanBuilder::from(intermediate_plan)
+ .project(inner_projection_exprs)?
+ .unnest_columns_with_options(unnest_col_vec, unnest_options)?
+ .build()?;
+ intermediate_expr_groups = outer_expr_groups;
+ }
}
fn get_struct_unnest_columns(
@@ -1017,6 +1174,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
having_expr_opt: Option<&Expr>,
qualify_expr_opt: Option<&Expr>,
order_by_exprs: &[SortExpr],
+ on_exprs: &[Expr],
group_by_exprs: &[Expr],
aggr_exprs: &[Expr],
) -> Result<AggregatePlanResult> {
@@ -1183,12 +1341,28 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
),
)?;
+ // Rewrite the DISTINCT ON expressions to use the columns produced by
+ // the aggregation. Same shape as ORDER BY rewriting so a hidden
+ // grouping column or a raw aggregate expression in ON is resolved.
+ let on_exprs_post_aggr = on_exprs
+ .iter()
+ .map(|expr| rebase_expr(expr, &aggr_projection_exprs, input))
+ .collect::<Result<Vec<Expr>>>()?;
+ check_columns_satisfy_exprs(
+ &all_valid_exprs,
+ &on_exprs_post_aggr,
+ CheckColumnsSatisfyExprsPurpose::Aggregate(
+ CheckColumnsMustReferenceAggregatePurpose::DistinctOn,
+ ),
+ )?;
+
Ok(AggregatePlanResult {
plan,
select_exprs: select_exprs_post_aggr,
having_expr: having_expr_post_aggr,
qualify_expr: qualify_expr_post_aggr,
order_by_exprs: order_by_post_aggr,
+ on_exprs: on_exprs_post_aggr,
})
}
diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs
index 1a76dd69f4..3b571eed27 100644
--- a/datafusion/sql/src/utils.rs
+++ b/datafusion/sql/src/utils.rs
@@ -35,7 +35,7 @@ use datafusion_expr::expr::{
};
use datafusion_expr::utils::{expr_as_column_expr, find_column_exprs};
use datafusion_expr::{
- ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, col, expr_vec_fmt,
+ ColumnUnnestList, Expr, ExprSchemable, LogicalPlan, SortExpr, col, expr_vec_fmt,
};
use indexmap::IndexMap;
@@ -98,6 +98,7 @@ pub(crate) enum CheckColumnsMustReferenceAggregatePurpose {
Having,
Qualify,
OrderBy,
+ DistinctOn,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -120,6 +121,9 @@ impl CheckColumnsSatisfyExprsPurpose {
Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::OrderBy) => {
"Column in ORDER BY must be in GROUP BY or an aggregate function"
}
+ Self::Aggregate(CheckColumnsMustReferenceAggregatePurpose::DistinctOn) => {
+ "Column in DISTINCT ON must be in GROUP BY or an aggregate function"
+ }
}
}
@@ -202,6 +206,42 @@ pub(crate) fn extract_aliases(exprs: &[Expr]) -> HashMap<String, Expr> {
.collect::<HashMap<String, Expr>>()
}
+/// If `expr` is a bare unqualified `Column` whose name matches a SELECT
+/// alias, swap it for the alias's underlying expression. Nested occurrences
+/// are left alone: PostgreSQL only resolves a top-level identifier as an
+/// output alias in clauses like ORDER BY and DISTINCT ON.
+pub(crate) fn substitute_top_level_alias(
+ expr: Expr,
+ aliases: &HashMap<String, Expr>,
+) -> Expr {
+ if let Expr::Column(col) = &expr
+ && col.relation.is_none()
+ && let Some(underlying) = aliases.get(&col.name)
+ {
+ return underlying.clone();
+ }
+
+ expr
+}
+
+/// Applies [`substitute_top_level_alias`] to each sort expression.
+pub(crate) fn substitute_top_level_aliases_in_sorts(
+ sort_exprs: Vec<SortExpr>,
+ aliases: &HashMap<String, Expr>,
+) -> Vec<SortExpr> {
+ if aliases.is_empty() {
+ return sort_exprs;
+ }
+
+ sort_exprs
+ .into_iter()
+ .map(|sort_expr| {
+ sort_expr
+ .with_expr(substitute_top_level_alias(sort_expr.expr.clone(), aliases))
+ })
+ .collect()
+}
+
/// Given an expression that's literal int encoding position, lookup the corresponding expression
/// in the select_exprs list, if the index is within the bounds and it is indeed a position literal,
/// otherwise, returns planning error.
diff --git a/datafusion/sqllogictest/test_files/distinct_on.slt b/datafusion/sqllogictest/test_files/distinct_on.slt
index 5b18915080..0659b9c208 100644
--- a/datafusion/sqllogictest/test_files/distinct_on.slt
+++ b/datafusion/sqllogictest/test_files/distinct_on.slt
@@ -195,3 +195,233 @@ RESET datafusion.explain.logical_plan_only;
statement ok
drop table t;
+
+# DISTINCT ON combined with GROUP BY + aggregation (issue #17256).
+# ON references a grouping column; ORDER BY uses an aggregate alias.
+query TII
+SELECT DISTINCT ON (c1) c1, c3, max(c4) AS agg2
+FROM aggregate_test_100 GROUP BY c1, c3 ORDER BY c1, agg2;
+----
+a 65 -28462
+b -60 -21739
+c 3 -30508
+d 102 -24558
+e -56 -31500
+
+# DISTINCT ON referencing a SELECT alias for an aggregate.
+query TI
+SELECT DISTINCT ON (agg2) c1, max(c4) AS agg2
+FROM aggregate_test_100 GROUP BY c1 ORDER BY agg2;
+----
+b 25286
+c 29106
+d 31106
+a 32064
+e 32514
+
+# DISTINCT ON with a scalar function over a grouping column.
+query TI
+SELECT DISTINCT ON (upper(c1)) c1, sum(c3) FROM aggregate_test_100
+GROUP BY c1 ORDER BY upper(c1);
+----
+a -385
+b -111
+c -28
+d 458
+e 847
+
+# Hidden ORDER BY tie-breaker: c3 is in GROUP BY and ORDER BY but
+# NOT in the SELECT list. PostgreSQL accepts this.
+query TI
+SELECT DISTINCT ON (c1) c1, max(c4) AS agg2
+FROM aggregate_test_100 GROUP BY c1, c3 ORDER BY c1, c3;
+----
+a 11640
+b 19316
+c -30187
+d 5613
+e 13611
+
+# Hidden DISTINCT ON key: ON references a grouping column that is
+# NOT in the SELECT list. ORDER BY adds a deterministic tie-breaker.
+query II
+SELECT DISTINCT ON (c1) c2 % 2, count(*) AS n
+FROM aggregate_test_100 GROUP BY c1, c2 % 2 ORDER BY c1, c2 % 2;
+----
+0 7
+0 9
+0 11
+0 6
+0 12
+
+# Raw aggregate expression in DISTINCT ON (not via an alias).
+query TI
+SELECT DISTINCT ON (sum(c3)) c1, sum(c3) AS total
+FROM aggregate_test_100 GROUP BY c1 ORDER BY sum(c3);
+----
+a -385
+b -111
+c -28
+d 458
+e 847
+
+# DISTINCT ON with HAVING.
+query TI
+SELECT DISTINCT ON (c1) c1, count(*) AS cnt FROM aggregate_test_100
+GROUP BY c1 HAVING count(*) > 10 ORDER BY c1, cnt DESC;
+----
+a 21
+b 19
+c 21
+d 18
+e 21
+
+# DISTINCT ON combined with a window function over a unique ordering
+# key, so the test is fully deterministic.
+query II
+WITH t(id, v) AS (VALUES (1, 10), (2, 20), (3, 10), (4, 30), (5, 20))
+SELECT DISTINCT ON (v) v, row_number() OVER (ORDER BY id) AS rn
+FROM t ORDER BY v, rn;
+----
+10 1
+20 2
+30 4
+
+# Raw window expression in DISTINCT ON (not via an alias). Uses a
+# unique ordering key so row_number is deterministic.
+query II
+WITH t(id, v) AS (VALUES (1, 10), (2, 20), (3, 30), (4, 40))
+SELECT DISTINCT ON (row_number() OVER (ORDER BY id)) id, v
+FROM t
+ORDER BY row_number() OVER (ORDER BY id);
+----
+1 10
+2 20
+3 30
+4 40
+
+# Qualified join columns with potential alias conflict.
+query TII
+WITH t1(k, v) AS (VALUES ('x', 1), ('x', 2), ('y', 3)),
+ t2(k, w) AS (VALUES ('x', 10), ('y', 20))
+SELECT DISTINCT ON (t1.k) t1.k, sum(t1.v) AS s, max(t2.w) AS mw
+FROM t1 JOIN t2 ON t1.k = t2.k
+GROUP BY t1.k ORDER BY t1.k;
+----
+x 3 10
+y 3 20
+
+# DISTINCT ON name conflicts with an input column of the same name.
+# PostgreSQL resolves `b` to the SELECT alias `a AS b`, not the input
+# column `t.b`. Groups should be keyed by `a`, not `t.b`.
+query TI
+WITH t(a, b) AS (VALUES ('x', 1), ('x', 2), ('y', 1))
+SELECT DISTINCT ON (b) a AS b, count(*) AS n
+FROM t GROUP BY a ORDER BY b;
+----
+x 2
+y 1
+
+# A bare alias resolves to the SELECT expression, but inside a larger
+# expression the same identifier refers to the input column. Postgres:
+# ORDER BY a, b + 0 DESC
+# uses `a` (post-aggregate) and `t.b + 0`, so for a=100 the row with
+# t.b=2 wins (sum=2). DataFusion must not recursively swap `b` inside
+# `b + 0` for the alias.
+query II
+WITH t(a, b) AS (VALUES (100, 1), (100, 2), (200, 1))
+SELECT DISTINCT ON (a) a AS b, sum(b) AS s
+FROM t GROUP BY a, b ORDER BY a, b + 0 DESC;
+----
+100 2
+200 1
+
+# A nested ORDER BY expression over a SELECT alias is still rejected in
+# the post-aggregate DISTINCT ON path.
+query error No field named agg2
+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))
+SELECT DISTINCT ON (a) a, max(b) AS agg2
+FROM t GROUP BY a ORDER BY a, agg2 + 1;
+
+# DISTINCT ON after aggregation still needs the SELECT-list unnest
+# rewrite, and bare ORDER BY aliases should keep working against the
+# rewritten DistinctOn input.
+query II
+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))
+SELECT DISTINCT ON (a) a, unnest(array_agg(b)) AS b
+FROM t GROUP BY a ORDER BY a, b DESC;
+----
+1 20
+2 30
+
+# DISTINCT ON after aggregation also needs to keep multi-column SELECT
+# expansions from struct unnest working.
+query III
+WITH t(a, b) AS (VALUES (1, 10), (1, 20), (2, 30))
+SELECT DISTINCT ON (a) a, unnest(struct(max(b), min(b)))
+FROM t GROUP BY a ORDER BY a;
+----
+1 20 10
+2 30 30
+
+# DISTINCT ON keys that expand to multiple columns are rejected.
+query error DISTINCT ON expressions that expand to multiple columns are not supported with DISTINCT ON
+WITH t(a, b) AS (VALUES (1, 10), (2, 20))
+SELECT DISTINCT ON (unnest(struct(max(b), min(b)))) a
+FROM t GROUP BY a ORDER BY unnest(struct(max(b), min(b))), a;
+
+# ORDER BY tie-breakers that expand to multiple columns are rejected.
+query error ORDER BY expressions that expand to multiple columns are not supported with DISTINCT ON
+WITH t(a, b) AS (VALUES (1, 10), (2, 20))
+SELECT DISTINCT ON (a) a, max(b)
+FROM t GROUP BY a ORDER BY a, unnest(struct(max(b), min(b)));
+
+# Fast path (no aggregation): a bare ORDER BY alias must resolve to
+# its underlying SELECT expression so that the sort attached to
+# DistinctOn normalizes against the base plan.
+query T
+WITH t(a, b) AS (VALUES ('x', 1), ('x', 2), ('y', 3))
+SELECT DISTINCT ON (x) a AS x FROM t ORDER BY x;
+----
+x
+y
+
+# EXPLAIN for the post-aggregation case.
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+query TT
+explain SELECT DISTINCT ON (c1) c1, max(c4) AS agg2
+FROM aggregate_test_100 GROUP BY c1 ORDER BY c1, agg2;
+----
+logical_plan
+01)Projection: first_value(aggregate_test_100.c1) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST] AS c1, first_value(agg2) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST] AS agg2
+02)--Sort: aggregate_test_100.c1 ASC NULLS LAST
+03)----Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[first_value(aggregate_test_100.c1) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST], first_value(max(aggregate_test_100.c4) AS agg2) ORDER BY [aggregate_test_100.c1 ASC NULLS LAST, max(aggregate_test_100.c4) ASC NULLS LAST]]]
+04)------Aggregate: groupBy=[[aggregate_test_100.c1]], aggr=[[max(aggregate_test_100.c4)]]
+05)--------TableScan: aggregate_test_100 projection=[c1, c4]
+
+statement ok
+RESET datafusion.explain.logical_plan_only;
+
+# Ordinal ORDER BY still works in the post-aggregate DISTINCT ON path.
+query TI
+SELECT DISTINCT ON (c1) c1, max(c4)
+FROM aggregate_test_100 GROUP BY c1 ORDER BY 1, 2;
+----
+a 32064
+b 25286
+c 29106
+d 31106
+e 32514
+
+# Synthetic repro for issue #17256.
+query TIR
+WITH t(a, b, c) AS (
+ VALUES ('x', 1, 10.0), ('x', 1, 20.0), ('y', 2, 30.0)
+)
+SELECT DISTINCT ON (a) a, b, sum(c) AS total
+FROM t GROUP BY a, b ORDER BY a, total DESC;
+----
+x 1 30
+y 2 30