Apache DataFusion SQL Query Engine
Perf: Take the comparison-free fast path in the grouped first/last accumulator when input is pre-ordered (#25050)
Qi Zhu4cee381898d0
diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rsindex d32078ec63..2b2d2483d7 100644--- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs+++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/query_builder.rs@@ -293,20 +293,51 @@ impl QueryBuilder {while aggregate_functions.len() < num_aggregate_functions {let idx = rng.random_range(0..self.aggregate_functions.len());let (function_name, is_distinct) = &self.aggregate_functions[idx];- let argument = self.random_argument();let alias = format!("col{alias_gen}");let distinct = if *is_distinct { "DISTINCT " } else { "" };alias_gen += 1;- let (order_by, null_opt) = if function_name.eq("first_value")+ let (argument, order_by, null_opt) = if function_name.eq("first_value")|| function_name.eq("last_value"){- (- self.order_by(&order_by_black_list), /* Among the order by columns, at most one group by column can be included to avoid all order by column values being identical */- self.null_opt(),- )+ if !self.dataset_sort_keys.is_empty() && rng.random_bool(0.5) {+ // Draw the ORDER BY from a prefix of the dataset's own+ // sort keys, so on the sorted datasets the plan satisfies+ // the aggregate's requirement and the pre-ordered+ // first/last fast path is exercised; the unsorted dataset+ // runs the same query through the comparing path, giving+ // a hinted-vs-unhinted comparison for free. A random+ // ORDER BY never does this: it samples up to 12 columns+ // with random directions against at most 3 sort keys.+ //+ // The argument is the last ORDER BY column on purpose:+ // rows that tie on the whole ORDER BY then carry equal+ // output values, so the documented tie-break difference+ // between the fast path (physically last/first) and the+ // tournament (first seen) cannot produce false+ // mismatches.+ let keys = &self.dataset_sort_keys+ [rng.random_range(0..self.dataset_sort_keys.len())];+ let prefix_len = rng.random_range(1..=keys.len());+ let prefix = &keys[..prefix_len];+ let order_by = format!(+ " order by {}",+ prefix+ .iter()+ .map(|c| format!("{c} ASC"))+ .collect::<Vec<_>>()+ .join(", ")+ );+ (prefix[prefix_len - 1].clone(), order_by, self.null_opt())+ } else {+ (+ self.random_argument(),+ self.order_by(&order_by_black_list), /* Among the order by columns, at most one group by column can be included to avoid all order by column values being identical */+ self.null_opt(),+ )+ }} else {- ("".to_string(), "".to_string())+ (self.random_argument(), "".to_string(), "".to_string())};let function = format!(diff --git a/datafusion/core/tests/physical_optimizer/mod.rs b/datafusion/core/tests/physical_optimizer/mod.rsindex f3b2884dab..f89d86074b 100644--- a/datafusion/core/tests/physical_optimizer/mod.rs+++ b/datafusion/core/tests/physical_optimizer/mod.rs@@ -38,6 +38,7 @@ mod replace_with_order_preserving_variants;mod sanity_checker;#[expect(clippy::needless_pass_by_value)]mod test_utils;+mod update_aggr_exprs;mod window_optimize;mod window_topn;diff --git a/datafusion/core/tests/physical_optimizer/update_aggr_exprs.rs b/datafusion/core/tests/physical_optimizer/update_aggr_exprs.rsnew file mode 100644index 0000000000..d2cdbd1761--- /dev/null+++ b/datafusion/core/tests/physical_optimizer/update_aggr_exprs.rs@@ -0,0 +1,144 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Tests for the [`OptimizeAggregateOrder`] rule.++use std::sync::Arc;++use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion_common::Result;+use datafusion_common::config::ConfigOptions;+use datafusion_functions_aggregate::first_last::first_value_udaf;+use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};+use datafusion_physical_expr::expressions::col;+use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_optimizer::PhysicalOptimizerRule;+use datafusion_physical_optimizer::update_aggr_exprs::OptimizeAggregateOrder;+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::aggregates::{+ AggregateExec, AggregateMode, PhysicalGroupBy,+};++use crate::physical_optimizer::test_utils::parquet_exec_with_sort;++fn abc_schema() -> SchemaRef {+ Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ Field::new("c", DataType::Int32, false),+ Field::new("d", DataType::Int32, false),+ ]))+}++/// An input whose declared ordering is `(a, b, c)`.+fn ordered_input(schema: &SchemaRef) -> Result<Arc<dyn ExecutionPlan>> {+ let ordering: LexOrdering = [+ PhysicalSortExpr::new_default(col("a", schema)?),+ PhysicalSortExpr::new_default(col("b", schema)?),+ PhysicalSortExpr::new_default(col("c", schema)?),+ ]+ .into();+ Ok(parquet_exec_with_sort(Arc::clone(schema), vec![ordering]) as _)+}++/// `FIRST_VALUE(d ORDER BY c)`.+fn first_value_expr(schema: &SchemaRef) -> Result<Arc<AggregateFunctionExpr>> {+ let order = PhysicalSortExpr::new_default(col("c", schema)?);+ AggregateExprBuilder::new(first_value_udaf(), vec![col("d", schema)?])+ .schema(Arc::clone(schema))+ .alias("first_value(d)")+ .order_by(vec![order])+ .build()+ .map(Arc::new)+}++fn optimized_flag(plan: Arc<dyn ExecutionPlan>) -> Result<String> {+ let optimized =+ OptimizeAggregateOrder::new().optimize(plan, &ConfigOptions::default())?;+ let agg = optimized+ .downcast_ref::<AggregateExec>()+ .expect("optimizer keeps the AggregateExec at the root");+ Ok(format!("{:?}", agg.aggr_expr()[0].fun().inner()))+}++/// With a plain GROUP BY, the input ordering `(a, b, c)` proves the group-by+/// prefix plus the aggregate's `ORDER BY c`, so the rule marks the aggregate+/// pre-ordered.+#[test]+fn single_group_by_gets_the_group_by_prefix() -> Result<()> {+ let schema = abc_schema();+ let group_by = PhysicalGroupBy::new_single(vec![+ (col("a", &schema)?, "a".to_string()),+ (col("b", &schema)?, "b".to_string()),+ ]);+ let agg = AggregateExec::try_new(+ AggregateMode::Single,+ group_by,+ vec![first_value_expr(&schema)?],+ vec![None],+ ordered_input(&schema)?,+ Arc::clone(&schema),+ )?;+ let dbg = optimized_flag(Arc::new(agg))?;+ assert!(+ dbg.contains("is_input_pre_ordered: true"),+ "expected the flag set for a single grouping set, got: {dbg}"+ );+ Ok(())+}++/// With grouping sets the same rows are fed once per grouping set, and within+/// a coarser set's group the rows follow the full group-by prefix rather than+/// the aggregate's own ORDER BY. The rule must not use the group-by prefix to+/// prove the requirement there.+#[test]+fn grouping_sets_do_not_get_the_group_by_prefix() -> Result<()> {+ let schema = abc_schema();+ // ROLLUP(a, b): grouping sets (a, b), (a), ().+ let group_by = PhysicalGroupBy::new(+ vec![+ (col("a", &schema)?, "a".to_string()),+ (col("b", &schema)?, "b".to_string()),+ ],+ vec![+ (+ datafusion_physical_expr::expressions::lit(1i32),+ "a".to_string(),+ ),+ (+ datafusion_physical_expr::expressions::lit(1i32),+ "b".to_string(),+ ),+ ],+ vec![vec![false, false], vec![false, true], vec![true, true]],+ true,+ );+ let agg = AggregateExec::try_new(+ AggregateMode::Single,+ group_by,+ vec![first_value_expr(&schema)?],+ vec![None],+ ordered_input(&schema)?,+ Arc::clone(&schema),+ )?;+ let dbg = optimized_flag(Arc::new(agg))?;+ assert!(+ dbg.contains("is_input_pre_ordered: false"),+ "the flag must stay unset under grouping sets, got: {dbg}"+ );+ Ok(())+}diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex d36ea5d630..f2d6af5bcf 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -79,6 +79,7 @@ pub fn last_value(expression: Expr, order_by: Vec<SortExpr>) -> Expr {fn create_groups_accumulator_helper<S: ValueState + 'static>(args: &AccumulatorArgs,is_first: bool,+ is_input_pre_ordered: bool,state: S,) -> Result<Box<dyn GroupsAccumulator>> {let Some(ordering) = LexOrdering::new(args.order_bys.to_vec()) else {@@ -96,12 +97,14 @@ fn create_groups_accumulator_helper<S: ValueState + 'static>(args.ignore_nulls,&ordering_dtypes,is_first,+ is_input_pre_ordered,)?))}fn create_groups_accumulator(args: &AccumulatorArgs,is_first: bool,+ is_input_pre_ordered: bool,function_name: &str,) -> Result<Box<dyn GroupsAccumulator>> {let data_type = args.return_field.data_type();@@ -111,6 +114,7 @@ fn create_groups_accumulator(create_groups_accumulator_helper(args,is_first,+ is_input_pre_ordered,PrimitiveValueState::<$t>::new(data_type.clone()),)};@@ -168,6 +172,7 @@ fn create_groups_accumulator(| DataType::BinaryView => create_groups_accumulator_helper(args,is_first,+ is_input_pre_ordered,BytesValueState::try_new(data_type.clone())?,),@@ -185,6 +190,7 @@ fn create_groups_accumulator(| DataType::Map(_, _) => create_groups_accumulator_helper(args,is_first,+ is_input_pre_ordered,GenericValueState::new(data_type.clone()),),@@ -346,7 +352,7 @@ impl AggregateUDFImpl for FirstValue {&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {- create_groups_accumulator(&args, true, self.name())+ create_groups_accumulator(&args, true, self.is_input_pre_ordered, self.name())}fn with_beneficial_ordering(@@ -393,10 +399,25 @@ struct FirstLastGroupsAccumulator<S: ValueState> {// to avoid calling `ScalarValue::size_of_vec` by Self.size.size_of_orderings: usize,- // buffer for `get_filtered_extreme_of_each_group`- // filter_min_of_each_group_buf.0[group_idx] -> idx_in_val- // only valid if filter_min_of_each_group_buf.1[group_idx] == true+ // Per-batch scoreboard shared by `get_filtered_extreme_of_each_group`+ // and the pre-ordered fast path:+ // extreme_of_each_group_buf.0[group_idx] -> idx_in_val+ // only valid if extreme_of_each_group_buf.1[group_idx] == trueextreme_of_each_group_buf: (Vec<usize>, BooleanBufferBuilder),+ // Set by `get_filtered_extreme_of_each_group` (merge_batch /+ // convert_to_state), which clears the scoreboard at its start but leaves+ // its winners' bits set on return. `update_batch_pre_ordered` keeps the+ // scoreboard all-false between its own batches, but the two can+ // interleave on one instance (the aggregation stream calls merge_batch on+ // the same accumulators when replaying spilled state), so the fast path+ // does one full reset when this is set.+ extreme_buf_dirty: bool,+ // Batch-local list of groups touched by `update_batch_pre_ordered`,+ // reused across batches. It lets winner collection and the scoreboard+ // reset run in O(groups touched by the batch) instead of+ // O(total_num_groups): with 1M allocated groups and 64 touched per+ // batch, a full-scoreboard sweep per batch dominates the runtime.+ touched_groups_buf: Vec<usize>,// =========== option ============@@ -409,6 +430,11 @@ struct FirstLastGroupsAccumulator<S: ValueState> {sort_options: Vec<SortOptions>,// Ignore null values.ignore_nulls: bool,+ // When `true` (set through `with_beneficial_ordering` by the+ // `OptimizeAggregateOrder` physical-optimizer rule), the optimizer has+ // proven that every group's rows already arrive in `ordering_req` order,+ // and `update_batch` takes the comparison-free fast path.+ is_input_pre_ordered: bool,default_orderings: Vec<ScalarValue>,}@@ -419,6 +445,7 @@ impl<S: ValueState> FirstLastGroupsAccumulator<S> {ignore_nulls: bool,ordering_dtypes: &[DataType],pick_first_in_group: bool,+ is_input_pre_ordered: bool,) -> Result<Self> {let default_orderings = ordering_dtypes.iter()@@ -437,7 +464,10 @@ impl<S: ValueState> FirstLastGroupsAccumulator<S> {is_sets: BooleanBufferBuilder::new(0),size_of_orderings: 0,extreme_of_each_group_buf: (Vec::new(), BooleanBufferBuilder::new(0)),+ extreme_buf_dirty: false,+ touched_groups_buf: Vec::new(),pick_first_in_group,+ is_input_pre_ordered,})}@@ -553,14 +583,14 @@ impl<S: ValueState> FirstLastGroupsAccumulator<S> {vals: &ArrayRef,is_set_arr: Option<&BooleanArray>,) -> Result<Vec<(usize, usize)>> {- // Set all values in min_of_each_group_buf.1 to false.+ // Set all values in extreme_of_each_group_buf.1 to false.self.extreme_of_each_group_buf.1.truncate(0);self.extreme_of_each_group_buf.1.append_n(self.is_sets.len(), false);- // No need to call `clear` since `self.min_of_each_group_buf.0[group_idx]`- // is only valid when `self.min_of_each_group_buf.1[group_idx] == true`.+ // No need to call `clear` since `self.extreme_of_each_group_buf.0[group_idx]`+ // is only valid when `self.extreme_of_each_group_buf.1[group_idx] == true`.let comparator = {assert_eq!(orderings.len(), self.ordering_req.len());@@ -614,6 +644,11 @@ impl<S: ValueState> FirstLastGroupsAccumulator<S> {}}+ // Winners' bits stay set on return; tell the pre-ordered fast path+ // that the scoreboard needs a reset before it can trust its all-false+ // invariant again.+ self.extreme_buf_dirty = true;+Ok(self.extreme_of_each_group_buf.0@@ -623,6 +658,108 @@ impl<S: ValueState> FirstLastGroupsAccumulator<S> {.map(|(group_idx, idx_in_val)| (group_idx, *idx_in_val)).collect::<Vec<_>>())}++ /// Comparison-free `update_batch` for pre-ordered input.+ ///+ /// `is_input_pre_ordered` is only set (through `with_beneficial_ordering`,+ /// by the `OptimizeAggregateOrder` physical-optimizer rule) after the+ /// optimizer has proven that the input ordering satisfies the ordered+ /// group-by prefix followed by this aggregate's own ordering requirement.+ /// Under that guarantee each group's rows arrive in `ordering_req` order,+ /// so:+ /// - within a batch, the extreme row of a group is simply its first+ /// (FIRST_VALUE) or last (LAST_VALUE) qualifying row, and+ /// - across batches, a later batch's winner always beats an earlier one+ /// for LAST_VALUE, and never does for FIRST_VALUE.+ ///+ /// No lexicographic comparator is built and `compare_rows` never runs.+ /// The winner's ordering values are still materialized into+ /// `self.orderings`: partial state must carry them, because the final+ /// aggregation stage merges states coming from different partitions whose+ /// relative order is not guaranteed, so `merge_batch` keeps comparing.+ ///+ /// Tie handling: among rows whose ordering keys compare equal, this path+ /// picks the physically last qualifying row for LAST_VALUE (and the first+ /// for FIRST_VALUE), matching the single-group pre-ordered accumulator and+ /// `Iterator::max_by`. The tournament path keeps the first-seen row of a+ /// tie instead (its comparisons are strict). Both are valid answers —+ /// which row of a tie wins is unspecified — but results can differ on+ /// tied keys.+ fn update_batch_pre_ordered(+ &mut self,+ values_and_order_cols: &[ArrayRef],+ group_indices: &[usize],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<()> {+ let vals = &values_and_order_cols[0];++ // `extreme_of_each_group_buf.1` is sized by `resize_states` and kept+ // all-false between batches: each batch records the groups it touched+ // in `touched_groups_buf` and clears exactly those bits before+ // returning. Neither the reset nor the winner collection may scan+ // `total_num_groups` -- with 1M allocated groups and 64 touched, a+ // full sweep turns a ~13us batch into ~400us and erases the win.+ debug_assert!(self.touched_groups_buf.is_empty());+ if self.extreme_buf_dirty {+ // A merge (spill replay) ran on this instance and left its+ // winners' bits set; restore the all-false invariant once.+ self.extreme_of_each_group_buf.1.truncate(0);+ self.extreme_of_each_group_buf+ .1+ .append_n(self.is_sets.len(), false);+ self.extreme_buf_dirty = false;+ }++ for (idx_in_val, &group_idx) in group_indices.iter().enumerate() {+ // A row passes the FILTER clause only when the predicate is+ // `true`; rows whose predicate evaluates to `null` are excluded.+ let passed_filter =+ opt_filter.is_none_or(|x| x.is_valid(idx_in_val) && x.value(idx_in_val));+ if !passed_filter {+ continue;+ }+ if self.ignore_nulls && vals.is_null(idx_in_val) {+ continue;+ }++ let touched_this_batch = self.extreme_of_each_group_buf.1.get_bit(group_idx);+ if self.pick_first_in_group+ && (self.is_sets.get_bit(group_idx) || touched_this_batch)+ {+ // The first qualifying row wins; groups decided by an earlier+ // batch (or earlier in this batch) never change again.+ continue;+ }+ if !touched_this_batch {+ self.extreme_of_each_group_buf.1.set_bit(group_idx, true);+ self.touched_groups_buf.push(group_idx);+ }+ // For LAST_VALUE, later qualifying rows unconditionally overwrite.+ self.extreme_of_each_group_buf.0[group_idx] = idx_in_val;+ }++ let mut ordering_buf = Vec::with_capacity(self.ordering_req.len());+ // Take the buffer to appease the borrow checker; `update_state`+ // needs `&mut self`.+ let touched = std::mem::take(&mut self.touched_groups_buf);+ for &group_idx in &touched {+ let idx = self.extreme_of_each_group_buf.0[group_idx];+ extract_row_at_idx_to_buf(+ &values_and_order_cols[1..],+ idx,+ &mut ordering_buf,+ )?;+ self.update_state(group_idx, &ordering_buf, vals, idx)?;+ }+ // Restore the all-false invariant by clearing only the touched bits.+ for &group_idx in &touched {+ self.extreme_of_each_group_buf.1.set_bit(group_idx, false);+ }+ self.touched_groups_buf = touched;+ self.touched_groups_buf.clear();++ Ok(())+ }}impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S> {@@ -636,6 +773,14 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S) -> Result<()> {self.resize_states(total_num_groups);+ if self.is_input_pre_ordered {+ return self.update_batch_pre_ordered(+ values_and_order_cols,+ group_indices,+ opt_filter,+ );+ }+let vals = &values_and_order_cols[0];let mut ordering_buf = Vec::with_capacity(self.ordering_req.len());@@ -1147,7 +1292,7 @@ impl AggregateUDFImpl for LastValue {&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {- create_groups_accumulator(&args, false, self.name())+ create_groups_accumulator(&args, false, self.is_input_pre_ordered, self.name())}}@@ -1587,6 +1732,7 @@ mod tests {true,&[DataType::Int64],true,+ false,)?;let mut val_with_orderings = {@@ -1679,6 +1825,7 @@ mod tests {true,&[DataType::Int64],true,+ false,)?;let val_with_orderings = {@@ -1760,6 +1907,7 @@ mod tests {true,&[DataType::Int64],false,+ false,)?;let mut val_with_orderings = {@@ -1834,6 +1982,7 @@ mod tests {true,&[DataType::Int64],true,+ false,)?;let val_with_orderings: Vec<ArrayRef> = vec![@@ -1888,6 +2037,7 @@ mod tests {true,&[DataType::Int64],true,+ false,)?;let val_with_orderings: Vec<ArrayRef> = vec![@@ -1910,6 +2060,7 @@ mod tests {true,&[DataType::Int64],true,+ false,)?;merging_acc.merge_batch(&state, &[0, 0], 1)?;@@ -2104,6 +2255,7 @@ mod tests {false,&[DataType::Int64],/* pick_first = */ true,+ false,)?;// Batch 1: four rows across two groups.@@ -2187,6 +2339,7 @@ mod tests {false,&[DataType::Int64],true,+ false,)?;// 10 groups × 10_000 candidate rows per group (100_000 total). Each@@ -2277,6 +2430,7 @@ mod tests {false,&[DataType::Int64],true,+ false,)?;const GROUPS: usize = 4;@@ -2385,4 +2539,426 @@ mod tests {}Ok(())}++ // ==================== pre-ordered fast path (#24771) ====================++ use arrow::datatypes::{Field, Int64Type};++ /// Builds a grouped first/last accumulator over Int64 values with a+ /// two-column Int64 ordering requirement `(o1, o2)`.+ #[expect(clippy::fn_params_excessive_bools)]+ fn grouped_acc(+ pick_first: bool,+ ignore_nulls: bool,+ pre_ordered: bool,+ descending: bool,+ ) -> Result<FirstLastGroupsAccumulator<PrimitiveValueState<Int64Type>>> {+ let schema = Schema::new(vec![+ Field::new("v", DataType::Int64, true),+ Field::new("o1", DataType::Int64, true),+ Field::new("o2", DataType::Int64, true),+ ]);+ let options = SortOptions {+ descending,+ nulls_first: false,+ };+ let ordering = LexOrdering::new(vec![+ PhysicalSortExpr::new(col("o1", &schema)?, options),+ PhysicalSortExpr::new(col("o2", &schema)?, options),+ ])+ .unwrap();+ FirstLastGroupsAccumulator::try_new(+ PrimitiveValueState::<Int64Type>::new(DataType::Int64),+ ordering,+ ignore_nulls,+ &[DataType::Int64, DataType::Int64],+ pick_first,+ pre_ordered,+ )+ }++ /// `[value, o1, o2]` columns for one batch.+ fn vo_batch(vals: &[Option<i64>], o1: &[i64], o2: &[i64]) -> Vec<ArrayRef> {+ assert_eq!(vals.len(), o1.len());+ assert_eq!(vals.len(), o2.len());+ vec![+ Arc::new(Int64Array::from(vals.to_vec())) as ArrayRef,+ Arc::new(Int64Array::from(o1.to_vec())) as ArrayRef,+ Arc::new(Int64Array::from(o2.to_vec())) as ArrayRef,+ ]+ }++ type PreOrderedBatch = (Vec<ArrayRef>, Vec<usize>, Option<BooleanArray>, usize);++ /// Feeds identical batches through the tournament path and the pre-ordered+ /// fast path, then asserts that every emitted state column (value,+ /// ordering columns, and the is_set flags) is identical.+ fn assert_matches_tournament(+ pick_first: bool,+ ignore_nulls: bool,+ descending: bool,+ batches: &[PreOrderedBatch],+ ) -> Result<()> {+ let mut slow = grouped_acc(pick_first, ignore_nulls, false, descending)?;+ let mut fast = grouped_acc(pick_first, ignore_nulls, true, descending)?;+ for (cols, group_indices, filter, total_num_groups) in batches {+ slow.update_batch(cols, group_indices, filter.as_ref(), *total_num_groups)?;+ fast.update_batch(cols, group_indices, filter.as_ref(), *total_num_groups)?;+ }+ let slow_state = slow.state(EmitTo::All)?;+ let fast_state = fast.state(EmitTo::All)?;+ assert_eq!(slow_state.len(), fast_state.len());+ for (col_idx, (s, f)) in slow_state.iter().zip(fast_state.iter()).enumerate() {+ assert_eq!(+ s.to_data(),+ f.to_data(),+ "state column {col_idx} differs between tournament and \+ pre-ordered fast path"+ );+ }+ Ok(())+ }++ fn int64_values(arr: &ArrayRef) -> Vec<Option<i64>> {+ let arr = arr.as_primitive::<Int64Type>();+ (0..arr.len())+ .map(|i| arr.is_valid(i).then(|| arr.value(i)))+ .collect()+ }++ /// LAST_VALUE, ascending, groups interleaved (PartiallySorted-style shape:+ /// each group's own rows are ordered even though groups mix), duplicate+ /// `o1` broken by `o2`, one group appearing only in the second batch.+ #[test]+ fn pre_ordered_last_value_matches_tournament() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(+ &[Some(10), Some(20), Some(11), Some(12), Some(21)],+ &[1, 1, 2, 2, 3],+ &[1, 1, 1, 2, 1],+ ),+ vec![0, 1, 0, 0, 1],+ None,+ 2,+ ),+ (+ vo_batch(&[Some(13), Some(30), Some(22)], &[4, 1, 5], &[1, 1, 1]),+ vec![0, 2, 1],+ None,+ 3,+ ),+ ];+ assert_matches_tournament(false, false, false, &batches)?;++ // Explicit expected values, so both paths being wrong together is+ // caught too: group 0 last row is (o=4,1)->13, group 1 is (5,1)->22,+ // group 2 only saw (1,1)->30.+ let mut fast = grouped_acc(false, false, true, false)?;+ for (cols, gids, filter, total) in &batches {+ fast.update_batch(cols, gids, filter.as_ref(), *total)?;+ }+ let out = fast.evaluate(EmitTo::All)?;+ assert_eq!(int64_values(&out), vec![Some(13), Some(22), Some(30)]);+ Ok(())+ }++ /// FIRST_VALUE, ascending: the first batch decides every group it saw;+ /// later batches must not overwrite.+ #[test]+ fn pre_ordered_first_value_locks_after_first_qualifying_row() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(&[Some(10), Some(20)], &[1, 1], &[1, 2]),+ vec![0, 1],+ None,+ 2,+ ),+ (+ vo_batch(&[Some(99), Some(98), Some(30)], &[2, 3, 1], &[1, 1, 1]),+ vec![0, 1, 2],+ None,+ 3,+ ),+ ];+ assert_matches_tournament(true, false, false, &batches)?;++ let mut fast = grouped_acc(true, false, true, false)?;+ for (cols, gids, filter, total) in &batches {+ fast.update_batch(cols, gids, filter.as_ref(), *total)?;+ }+ let out = fast.evaluate(EmitTo::All)?;+ assert_eq!(int64_values(&out), vec![Some(10), Some(20), Some(30)]);+ Ok(())+ }++ /// Descending ordering requirement with input laid out descending: the+ /// physically-last row is still the requirement's extreme.+ #[test]+ fn pre_ordered_descending_matches_tournament() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(+ &[Some(1), Some(2), Some(3), Some(4)],+ &[9, 9, 7, 5],+ &[5, 3, 1, 1],+ ),+ vec![0, 0, 1, 0],+ None,+ 2,+ ),+ (+ vo_batch(&[Some(5), Some(6)], &[4, 2], &[9, 9]),+ vec![0, 1],+ None,+ 2,+ ),+ ];+ assert_matches_tournament(false, false, true, &batches)?;+ assert_matches_tournament(true, false, true, &batches)?;+ Ok(())+ }++ /// FILTER interaction: a row with a `null` predicate is excluded; a group+ /// whose rows are all filtered in the last batch keeps its earlier winner;+ /// a group filtered everywhere stays unset (emits null + is_set=false).+ #[test]+ fn pre_ordered_respects_filter() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(+ &[Some(10), Some(11), Some(20), Some(30)],+ &[1, 2, 1, 1],+ &[1, 1, 1, 1],+ ),+ vec![0, 0, 1, 2],+ Some(BooleanArray::from(vec![+ Some(true),+ Some(true),+ Some(true),+ Some(false),+ ])),+ 3,+ ),+ (+ vo_batch(&[Some(12), Some(21), Some(31)], &[3, 2, 2], &[1, 1, 1]),+ vec![0, 1, 2],+ Some(BooleanArray::from(vec![Some(false), None, Some(false)])),+ 3,+ ),+ ];+ assert_matches_tournament(false, false, false, &batches)?;++ let mut fast = grouped_acc(false, false, true, false)?;+ for (cols, gids, filter, total) in &batches {+ fast.update_batch(cols, gids, filter.as_ref(), *total)?;+ }+ let state = fast.state(EmitTo::All)?;+ // value column: group 0 keeps batch-1 winner 11 (batch 2 filtered),+ // group 1 keeps 20 (null predicate excluded), group 2 never set.+ assert_eq!(int64_values(&state[0]), vec![Some(11), Some(20), None]);+ let is_sets = state.last().unwrap().as_boolean();+ assert_eq!(+ (0..3).map(|i| is_sets.value(i)).collect::<Vec<_>>(),+ vec![true, true, false]+ );+ Ok(())+ }++ /// IGNORE NULLS: null values are skipped, so the winner is the last+ /// non-null row; an all-null group stays unset.+ #[test]+ fn pre_ordered_respects_ignore_nulls() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(+ &[Some(10), None, None, Some(20)],+ &[1, 2, 1, 1],+ &[1, 1, 1, 2],+ ),+ vec![0, 0, 1, 1],+ None,+ 2,+ ),+ (+ vo_batch(&[None, None], &[3, 2], &[1, 1]),+ vec![0, 2],+ None,+ 3,+ ),+ ];+ assert_matches_tournament(false, true, false, &batches)?;+ assert_matches_tournament(true, true, false, &batches)?;++ let mut fast = grouped_acc(false, true, true, false)?;+ for (cols, gids, filter, total) in &batches {+ fast.update_batch(cols, gids, filter.as_ref(), *total)?;+ }+ let state = fast.state(EmitTo::All)?;+ assert_eq!(int64_values(&state[0]), vec![Some(10), Some(20), None]);+ Ok(())+ }++ /// Ties: among rows with equal ordering keys the fast path picks the+ /// physically last row for LAST_VALUE / first row for FIRST_VALUE,+ /// matching the single-group pre-ordered accumulator. (The tournament+ /// path keeps the first-seen row of a tie, so the two paths may pick+ /// different — equally valid — rows; this test pins the fast path's+ /// choice rather than asserting equivalence.)+ #[test]+ fn pre_ordered_tie_picks_positional_extreme() -> Result<()> {+ let cols = vo_batch(&[Some(10), Some(11), Some(12)], &[1, 1, 1], &[1, 1, 1]);+ let mut last = grouped_acc(false, false, true, false)?;+ last.update_batch(&cols, &[0, 0, 0], None, 1)?;+ assert_eq!(int64_values(&last.evaluate(EmitTo::All)?), vec![Some(12)]);++ let mut first = grouped_acc(true, false, true, false)?;+ first.update_batch(&cols, &[0, 0, 0], None, 1)?;+ assert_eq!(int64_values(&first.evaluate(EmitTo::All)?), vec![Some(10)]);+ Ok(())+ }++ /// RESPECT NULLS (the default): a null value can itself be the winner;+ /// the fast path must not treat value-nulls specially.+ #[test]+ fn pre_ordered_respect_nulls_null_can_win() -> Result<()> {+ let batches: Vec<PreOrderedBatch> = vec![+ (+ vo_batch(&[Some(10), None], &[1, 2], &[1, 1]),+ vec![0, 0],+ None,+ 1,+ ),+ (+ vo_batch(&[None, Some(7)], &[1, 2], &[1, 1]),+ vec![1, 1],+ None,+ 2,+ ),+ ];+ assert_matches_tournament(false, false, false, &batches)?;+ assert_matches_tournament(true, false, false, &batches)?;++ let mut fast = grouped_acc(false, false, true, false)?;+ for (cols, gids, filter, total) in &batches {+ fast.update_batch(cols, gids, filter.as_ref(), *total)?;+ }+ // Group 0's last row is the null; it wins under RESPECT NULLS.+ let out = fast.evaluate(EmitTo::All)?;+ assert_eq!(int64_values(&out), vec![None, Some(7)]);+ Ok(())+ }++ /// Draining with `EmitTo::First(n)` mid-stream shifts group indices; the+ /// fast path must stay in lockstep with the tournament path across the+ /// shift.+ #[test]+ fn pre_ordered_survives_partial_emit() -> Result<()> {+ let mut slow = grouped_acc(false, false, false, false)?;+ let mut fast = grouped_acc(false, false, true, false)?;++ let b1 = vo_batch(&[Some(10), Some(20), Some(30)], &[1, 1, 1], &[1, 1, 1]);+ for acc in [&mut slow, &mut fast] {+ acc.update_batch(&b1, &[0, 1, 2], None, 3)?;+ }++ // Emit the first two groups; group 2 shifts down to index 0.+ let s1 = slow.state(EmitTo::First(2))?;+ let f1 = fast.state(EmitTo::First(2))?;+ for (s, f) in s1.iter().zip(f1.iter()) {+ assert_eq!(s.to_data(), f.to_data());+ }+ assert_eq!(int64_values(&f1[0]), vec![Some(10), Some(20)]);++ // Keep feeding the surviving group (now index 0) plus a new group.+ let b2 = vo_batch(&[Some(31), Some(40)], &[2, 1], &[1, 1]);+ for acc in [&mut slow, &mut fast] {+ acc.update_batch(&b2, &[0, 1], None, 2)?;+ }+ let s2 = slow.state(EmitTo::All)?;+ let f2 = fast.state(EmitTo::All)?;+ for (s, f) in s2.iter().zip(f2.iter()) {+ assert_eq!(s.to_data(), f.to_data());+ }+ assert_eq!(int64_values(&f2[0]), vec![Some(31), Some(40)]);+ Ok(())+ }++ /// End-to-end partial→final: states produced by the fast path carry the+ /// winning ordering values, so a (never pre-ordered) final-stage+ /// accumulator merging two partitions in either arrival order picks the+ /// true global extreme.+ #[test]+ fn pre_ordered_partial_states_merge_correctly() -> Result<()> {+ let make_partition_state = |o1: i64, val: i64| -> Result<Vec<ArrayRef>> {+ let mut partial = grouped_acc(false, false, true, false)?;+ partial.update_batch(&vo_batch(&[Some(val)], &[o1], &[1]), &[0], None, 1)?;+ partial.state(EmitTo::All)+ };+ // Partition A saw the later row (o1=9), partition B the earlier one.+ let a = make_partition_state(9, 900)?;+ let b = make_partition_state(3, 300)?;++ for order in [[&a, &b], [&b, &a]] {+ let mut final_acc = grouped_acc(false, false, false, false)?;+ for state in order {+ final_acc.merge_batch(state, &[0], 1)?;+ }+ let out = final_acc.evaluate(EmitTo::All)?;+ assert_eq!(int64_values(&out), vec![Some(900)]);+ }+ Ok(())+ }++ /// Winner collection and scoreboard reset must scale with the groups a+ /// batch touches, not with `total_num_groups`. Correctness side of that:+ /// sparse high group indices against a large total still resolve.+ #[test]+ fn pre_ordered_sparse_groups_large_total() -> Result<()> {+ let total = 100_000;+ let mut fast = grouped_acc(false, false, true, false)?;+ fast.update_batch(+ &vo_batch(&[Some(10), Some(20)], &[1, 1], &[1, 1]),+ &[7, 42_000],+ None,+ total,+ )?;+ fast.update_batch(+ &vo_batch(&[Some(11), Some(30)], &[2, 1], &[1, 1]),+ &[7, 99_999],+ None,+ total,+ )?;+ let out = fast.evaluate(EmitTo::All)?;+ let vals = int64_values(&out);+ assert_eq!(vals.len(), total);+ assert_eq!(vals[7], Some(11));+ assert_eq!(vals[42_000], Some(20));+ assert_eq!(vals[99_999], Some(30));+ assert_eq!(vals[0], None);+ Ok(())+ }++ /// The aggregation stream calls `merge_batch` on the same accumulators+ /// when replaying spilled state, and the tournament helper leaves its+ /// winners' scoreboard bits set. A later pre-ordered `update_batch` must+ /// not mistake those for "touched in this batch", or a merged group's+ /// newer row is dropped from the winner list.+ #[test]+ fn pre_ordered_update_after_merge_interleave() -> Result<()> {+ let merged_state = {+ let mut partial = grouped_acc(false, false, true, false)?;+ partial.update_batch(&vo_batch(&[Some(100)], &[1], &[1]), &[0], None, 1)?;+ partial.state(EmitTo::All)?+ };++ let mut acc = grouped_acc(false, false, true, false)?;+ acc.merge_batch(&merged_state, &[0], 1)?;+ // A newer row (higher ordering key) for the merged group arrives+ // through the fast path afterwards.+ acc.update_batch(&vo_batch(&[Some(500)], &[5], &[1]), &[0], None, 1)?;+ let out = acc.evaluate(EmitTo::All)?;+ assert_eq!(int64_values(&out), vec![Some(500)]);+ Ok(())+ }}diff --git a/datafusion/physical-optimizer/src/update_aggr_exprs.rs b/datafusion/physical-optimizer/src/update_aggr_exprs.rsindex 2430918e2c..a047ab11f4 100644--- a/datafusion/physical-optimizer/src/update_aggr_exprs.rs+++ b/datafusion/physical-optimizer/src/update_aggr_exprs.rs@@ -89,20 +89,35 @@ impl PhysicalOptimizerRule for OptimizeAggregateOrder {let input = aggr_exec.input();let mut aggr_exprs = aggr_exec.aggr_expr().to_vec();- let groupby_exprs = aggr_exec.group_expr().input_exprs();// If the existing ordering satisfies a prefix of the GROUP BY// expressions, prefix requirements with this section. In this// case, aggregation will work more efficiently.- let indices = get_ordered_partition_by_indices(&groupby_exprs, input)?;- let requirement = indices- .iter()- .map(|&idx| {- PhysicalSortRequirement::new(- Arc::clone(&groupby_exprs[idx]),- None,- )- })- .collect::<Vec<_>>();+ //+ // Only do this for a single grouping set. With grouping sets+ // (e.g. ROLLUP), the stream feeds the same rows once per+ // grouping set, and within a coarser set's group the rows are+ // ordered by the *full* group-by prefix, not by the+ // aggregate's own ORDER BY. Proving the prefixed requirement+ // there would mark first/last aggregates as pre-ordered when+ // their groups are not, producing wrong results in positional+ // paths. Without the prefix, a globally satisfied ORDER BY+ // still holds within every group of every grouping set.+ let requirement = if aggr_exec.group_expr().is_single() {+ let groupby_exprs = aggr_exec.group_expr().input_exprs();+ let indices =+ get_ordered_partition_by_indices(&groupby_exprs, input)?;+ indices+ .iter()+ .map(|&idx| {+ PhysicalSortRequirement::new(+ Arc::clone(&groupby_exprs[idx]),+ None,+ )+ })+ .collect::<Vec<_>>()+ } else {+ vec![]+ };aggr_exprs = try_convert_aggregate_if_better(aggr_exprs,diff --git a/datafusion/sqllogictest/test_files/first_last_ordered.slt b/datafusion/sqllogictest/test_files/first_last_ordered.sltnew file mode 100644index 0000000000..63a7cc1cf3--- /dev/null+++ b/datafusion/sqllogictest/test_files/first_last_ordered.slt@@ -0,0 +1,183 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# End-to-end coverage for the grouped first_value/last_value pre-ordered fast+# path (issue #24771): when `OptimizeAggregateOrder` proves the input ordering+# satisfies the group-by prefix followed by the aggregate's ordering+# requirement, the grouped accumulator skips its per-row tournament. These+# queries exercise the whole chain — the requirement being proven through+# declared orderings, through projection-alias equivalences, and through+# filter-induced constants — and pin the results.++statement ok+CREATE EXTERNAL TABLE multiple_ordered_table (+ a0 INTEGER,+ a INTEGER,+ b INTEGER,+ c INTEGER,+ d INTEGER+)+STORED AS CSV+WITH ORDER (a ASC, b ASC)+WITH ORDER (c ASC)+LOCATION '../core/tests/data/window_2.csv'+OPTIONS ('format.has_header' 'true');++# Direct match: requirement (a) ++ (c ASC) is satisfied by the declared+# orderings, so the fast path runs.+query III+SELECT a, FIRST_VALUE(c ORDER BY c), LAST_VALUE(c ORDER BY c)+FROM multiple_ordered_table GROUP BY a ORDER BY a;+----+0 0 49+1 50 99++# The winner's non-key column rides along (argmax semantics).+query III+SELECT a, FIRST_VALUE(d ORDER BY c), LAST_VALUE(d ORDER BY c)+FROM multiple_ordered_table GROUP BY a ORDER BY a;+----+0 0 3+1 0 3++# Requirement proven through a projection-alias equivalence class: the input+# is ordered by (a, b) / (c), the query groups by an alias of `a`, and the+# aggregate orders by an alias of `c`.+query II+SELECT a2, LAST_VALUE(d2 ORDER BY c2)+FROM (SELECT a AS a2, c AS c2, d AS d2 FROM multiple_ordered_table)+GROUP BY a2 ORDER BY a2;+----+0 3+1 3++# Requirement proven with the help of a filter-induced constant: `a` is+# constant under `a = 1`, so the (b) prefix comes from the (a, b) ordering and+# `c` stays ordered inside every b-segment.+query III+SELECT b, FIRST_VALUE(c ORDER BY c), LAST_VALUE(c ORDER BY c)+FROM multiple_ordered_table WHERE a = 1 GROUP BY b ORDER BY b;+----+2 50 74+3 75 99++# FILTER clause interacts with the fast path.+query II+SELECT a, LAST_VALUE(c ORDER BY c) FILTER (WHERE c % 2 = 0)+FROM multiple_ordered_table GROUP BY a ORDER BY a;+----+0 48+1 98++# A DESC requirement over an ASC input takes the reverse branch: the rule+# swaps FIRST_VALUE for LAST_VALUE before setting the flag, and the result is+# unchanged.+query II+SELECT a, FIRST_VALUE(c ORDER BY c DESC)+FROM multiple_ordered_table GROUP BY a ORDER BY a;+----+0 49+1 99++statement ok+DROP TABLE multiple_ordered_table;++# With grouping sets the same rows are fed once per grouping set, and within a+# coarser set's group the rows are ordered by the full group-by prefix rather+# than the aggregate's ORDER BY. The rule must not mark first/last aggregates+# pre-ordered there (github.com/apache/datafusion/pull/25050 review): input+# below is ordered on (a, b, c) but within the ROLLUP's (a) set, group a=1+# arrives ordered by (b, c), so FIRST/LAST over `ORDER BY c` need comparisons.++statement ok+COPY (VALUES+ (1, 1, 5, 100, false), (1, 1, 6, 101, true),+ (1, 2, 1, 200, true), (1, 2, 2, 201, false))+TO 'test_files/scratch/first_last_ordered/rollup.csv'+OPTIONS ('format.has_header' 'false');++statement ok+CREATE EXTERNAL TABLE rollup_source (a INT, b INT, c INT, d INT, e BOOLEAN)+STORED AS CSV+LOCATION 'test_files/scratch/first_last_ordered/rollup.csv'+WITH ORDER (a ASC, b ASC, c ASC)+OPTIONS ('format.has_header' 'false');++query IIII+SELECT a, b, FIRST_VALUE(d ORDER BY c), LAST_VALUE(d ORDER BY c)+FROM rollup_source GROUP BY ROLLUP(a, b) ORDER BY a NULLS FIRST, b NULLS FIRST;+----+NULL NULL 200 101+1 NULL 200 101+1 1 100 101+1 2 200 201++# An aggregate ORDER BY that is satisfied without any group-by prefix stays on+# the fast path even under ROLLUP: `a` leads the input ordering, so it is+# ordered inside every group of every grouping set.+query III+SELECT a, b, LAST_VALUE(d ORDER BY a)+FROM rollup_source GROUP BY ROLLUP(a, b) ORDER BY a NULLS FIRST, b NULLS FIRST;+----+NULL NULL 201+1 NULL 201+1 1 101+1 2 201++# Boolean is outside groups_accumulator_supported, so these run through+# GroupsAccumulatorAdapter and the single-group accumulators, which consumed+# the pre-ordered flag on main already: this covers the latent wrong-results+# path that existed before this PR, not just the new grouped fast path.+query IIBB+SELECT a, b, FIRST_VALUE(e ORDER BY c), LAST_VALUE(e ORDER BY c)+FROM rollup_source GROUP BY ROLLUP(a, b) ORDER BY a NULLS FIRST, b NULLS FIRST;+----+NULL NULL true true+1 NULL true true+1 1 false true+1 2 true false++statement ok+DROP TABLE rollup_source;++# Tie handling is a user-visible behavior change: among rows whose ordering+# keys compare equal, the pre-ordered fast path picks the physically last+# qualifying row for LAST_VALUE (first for FIRST_VALUE), while the tournament+# path keeps the first-seen row of a tie. Both are valid answers -- which row+# of a tie wins is unspecified -- but declaring WITH ORDER on a table can+# change which one a query returns. This pins the fast path's choice.+statement ok+COPY (VALUES (1, 1, 10), (1, 1, 11), (2, 1, 20), (2, 1, 21))+TO 'test_files/scratch/first_last_ordered/ties.csv'+OPTIONS ('format.has_header' 'false');++statement ok+CREATE EXTERNAL TABLE tie_source (g INT, ts INT, px INT)+STORED AS CSV+LOCATION 'test_files/scratch/first_last_ordered/ties.csv'+WITH ORDER (g ASC, ts ASC)+OPTIONS ('format.has_header' 'false');++query III+SELECT g, FIRST_VALUE(px ORDER BY ts), LAST_VALUE(px ORDER BY ts)+FROM tie_source GROUP BY g ORDER BY g;+----+1 10 11+2 20 21++statement ok+DROP TABLE tie_source;