Apache DataFusion SQL Query Engine
refactor: remove `opt_filter` in `GroupsAccumulator::merge_batch` (#22816)
Huaijinc14379b9b528
diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rsindex f1651dbf28..b990740159 100644--- a/datafusion-examples/examples/udf/advanced_udaf.rs+++ b/datafusion-examples/examples/udf/advanced_udaf.rs@@ -268,7 +268,6 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&arrow::array::BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 2, "two arguments to merge_batch");@@ -280,7 +279,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {self.null_state.accumulate(group_indices,partial_counts,- opt_filter,+ None,total_num_groups,|group_index, partial_count| {self.counts[group_index] += partial_count;@@ -292,7 +291,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {self.null_state.accumulate(group_indices,partial_prods,- opt_filter,+ None,total_num_groups,|group_index, new_value: <Float64Type as ArrowPrimitiveType>::Native| {let prod = &mut self.prods[group_index];diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rsindex 7d22c5df70..b895cb9c7c 100644--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs@@ -872,7 +872,6 @@ impl GroupsAccumulator for TestGroupsAccumulator {&mut self,_values: &[ArrayRef],_group_indices: &[usize],- _opt_filter: Option<&arrow::array::BooleanArray>,_total_num_groups: usize,) -> Result<()> {Ok(())diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rsindex da5da384c7..b021674cbe 100644--- a/datafusion/expr-common/src/groups_accumulator.rs+++ b/datafusion/expr-common/src/groups_accumulator.rs@@ -183,12 +183,14 @@ pub trait GroupsAccumulator: Send + std::any::Any {////// * `values`: arrays produced from previously calling `state` on other accumulators.///- /// Other arguments are the same as for [`Self::update_batch`].+ /// Other arguments are the same as for [`Self::update_batch`], except that+ /// there is no `opt_filter` — aggregate filters are applied during the+ /// partial (update) phase, so by the time intermediate states are merged+ /// no per-row filtering is needed.fn merge_batch(&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()>;diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rsindex 1600bef39d..272afdb6ab 100644--- a/datafusion/ffi/src/udaf/groups_accumulator.rs+++ b/datafusion/ffi/src/udaf/groups_accumulator.rs@@ -64,7 +64,6 @@ pub struct FFI_GroupsAccumulator {accumulator: &mut Self,values: SVec<WrappedArray>,group_indices: SVec<usize>,- opt_filter: FFI_Option<WrappedArray>,total_num_groups: usize,) -> FFI_Result<()>,@@ -195,21 +194,14 @@ unsafe extern "C" fn merge_batch_fn_wrapper(accumulator: &mut FFI_GroupsAccumulator,values: SVec<WrappedArray>,group_indices: SVec<usize>,- opt_filter: FFI_Option<WrappedArray>,total_num_groups: usize,) -> FFI_Result<()> {unsafe {let accumulator = accumulator.inner_mut();let values = sresult_return!(process_values(values));let group_indices: Vec<usize> = group_indices.into_iter().collect();- let opt_filter = sresult_return!(process_opt_filter(opt_filter));- sresult!(accumulator.merge_batch(- &values,- &group_indices,- opt_filter.as_ref(),- total_num_groups- ))+ sresult!(accumulator.merge_batch(&values, &group_indices, total_num_groups))}}@@ -379,7 +371,6 @@ impl GroupsAccumulator for ForeignGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {unsafe {@@ -388,20 +379,11 @@ impl GroupsAccumulator for ForeignGroupsAccumulator {.map(WrappedArray::try_from).collect::<std::result::Result<Vec<_>, ArrowError>>()?;let group_indices = group_indices.iter().cloned().collect();- let opt_filter = opt_filter- .map(|bool_array| to_ffi(&bool_array.to_data()))- .transpose()?- .map(|(array, schema)| WrappedArray {- array,- schema: WrappedSchema(schema),- })- .into();df_result!((self.accumulator.merge_batch)(&mut self.accumulator,values.into_iter().collect(),group_indices,- opt_filter,total_num_groups))}@@ -517,8 +499,7 @@ mod tests {let second_states =vec![make_array(create_array!(Boolean, vec![false]).to_data())];- let opt_filter = create_array!(Boolean, vec![true]);- foreign_accum.merge_batch(&second_states, &[0], Some(opt_filter.as_ref()), 1)?;+ foreign_accum.merge_batch(&second_states, &[0], 1)?;let groups_bool = foreign_accum.evaluate(EmitTo::All)?;assert_eq!(groups_bool.len(), 1);assert_eq!(diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rsindex d370d59c90..60fe0388c4 100644--- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs@@ -160,7 +160,6 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> datafusion_common::Result<()> {debug_assert_eq!(values.len(), 1);diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rsindex ad2a21bb47..b412b4ffe0 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs@@ -375,13 +375,12 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {self.invoke_per_accumulator(values,group_indices,- opt_filter,+ None,total_num_groups,|accumulator, values_to_accumulate| {accumulator.merge_batch(values_to_accumulate)?;diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rsindex d1d8924a2c..afb1dec24a 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs@@ -132,11 +132,10 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// update / merge are the same- self.update_batch(values, group_indices, opt_filter, total_num_groups)+ self.update_batch(values, group_indices, None, total_num_groups)}fn size(&self) -> usize {diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rsindex a81b89e1e4..474899d8f3 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs@@ -131,11 +131,10 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// update / merge are the same- self.update_batch(values, group_indices, opt_filter, total_num_groups)+ self.update_batch(values, group_indices, None, total_num_groups)}/// Converts an input batch directly to a state batchdiff --git a/datafusion/functions-aggregate/benches/first_last.rs b/datafusion/functions-aggregate/benches/first_last.rsindex 1d18e1c7dc..8f28e126a4 100644--- a/datafusion/functions-aggregate/benches/first_last.rs+++ b/datafusion/functions-aggregate/benches/first_last.rs@@ -235,7 +235,6 @@ fn merge_bench(Arc::clone(&is_set),],&group_indices,- opt_filter,num_groups,).unwrap(),diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 38b902964f..0c5a438454 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -705,15 +705,8 @@ impl<H: HllValueHasher> GroupsAccumulator for HllGroupsAccumulator<H> {&mut self,values: &[ArrayRef],group_indices: &[usize],- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {- assert!(- opt_filter.is_none(),- "aggregate filter should be applied in partial stage, there should be no filter in final stage"- );-self.ensure_groups(total_num_groups);let states = downcast_value!(values[0], BinaryArray);let mut delta: isize = 0;diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rsindex 8ed3fbf8c3..1dd111f918 100644--- a/datafusion/functions-aggregate/src/array_agg.rs+++ b/datafusion/functions-aggregate/src/array_agg.rs@@ -732,7 +732,6 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "one argument to merge_batch");@@ -2019,7 +2018,7 @@ mod tests {// Merge acc2's state into acc1let state = acc2.state(EmitTo::All)?;- acc1.merge_batch(&state, &[0, 1], None, 2)?;+ acc1.merge_batch(&state, &[0, 1], 2)?;// Another update_batch on acc1 after the mergelet values: ArrayRef = Arc::new(Int32Array::from(vec![5, 6]));@@ -2088,7 +2087,7 @@ mod tests {// Feed state into a new accumulator via merge_batchlet mut acc2 = ArrayAggGroupsAccumulator::new(DataType::Int32, false);- acc2.merge_batch(&state, &[0, 0, 1], None, 2)?;+ acc2.merge_batch(&state, &[0, 0, 1], 2)?;// Group 0 received rows 0 ([1]) and 1 ([NULL]) → [1, NULL]let vals = eval_i32_lists(&mut acc2, EmitTo::All)?;@@ -2118,7 +2117,7 @@ mod tests {// Feed state into a new accumulator via merge_batchlet mut acc2 = ArrayAggGroupsAccumulator::new(DataType::Int32, true);- acc2.merge_batch(&state, &[0, 0, 1, 1], None, 2)?;+ acc2.merge_batch(&state, &[0, 0, 1, 1], 2)?;// Group 0: received [1] and null (skipped) → [1]let vals = eval_i32_lists(&mut acc2, EmitTo::All)?;diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rsindex ddeb9b0870..06c7694634 100644--- a/datafusion/functions-aggregate/src/average.rs+++ b/datafusion/functions-aggregate/src/average.rs@@ -908,7 +908,6 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 2, "two arguments to merge_batch");@@ -920,7 +919,7 @@ whereself.null_state.accumulate(group_indices,partial_counts,- opt_filter,+ None,total_num_groups,|group_index, partial_count| {// SAFETY: group_index is guaranteed to be in bounds@@ -934,7 +933,7 @@ whereself.null_state.accumulate(group_indices,partial_sums,- opt_filter,+ None,total_num_groups,|group_index, new_value: <T as ArrowPrimitiveType>::Native| {// SAFETY: group_index is guaranteed to be in boundsdiff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rsindex 5a95cfe832..7fcf4bb61f 100644--- a/datafusion/functions-aggregate/src/correlation.rs+++ b/datafusion/functions-aggregate/src/correlation.rs@@ -493,7 +493,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// Resize vectors to accommodate total number of groups@@ -512,11 +511,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {let partial_sum_xx = values[4].as_primitive::<Float64Type>();let partial_sum_yy = values[5].as_primitive::<Float64Type>();- assert!(- opt_filter.is_none(),- "aggregate filter should be applied in partial stage, there should be no filter in final stage"- );-accumulate_correlation_states(group_indices,(diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rsindex eab36d4951..f0ce8c82a1 100644--- a/datafusion/functions-aggregate/src/count.rs+++ b/datafusion/functions-aggregate/src/count.rs@@ -665,8 +665,6 @@ impl GroupsAccumulator for CountGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "one argument to merge_batch");diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex 1935f29c4c..cecb277cb8 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -671,7 +671,6 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {self.resize_states(total_num_groups);@@ -690,7 +689,7 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<Slet groups = self.get_filtered_extreme_of_each_group(&val_and_order_cols[1..],group_indices,- opt_filter,+ None,vals,Some(is_set_arr),)?;@@ -1587,12 +1586,7 @@ mod tests {group_acc.compute_size_of_orderings());- group_acc.merge_batch(- &state,- &[0, 1, 2],- Some(&BooleanArray::from(vec![true, false, false])),- 3,- )?;+ group_acc.merge_batch(&state, &[0, 1, 2], 3)?;assert_eq!(group_acc.size_of_orderings,@@ -1608,8 +1602,11 @@ mod tests {let binding = group_acc.evaluate(EmitTo::All)?;let eval_result = binding.as_any().downcast_ref::<Int64Array>().unwrap();+ // group 0 keeps merged value=1 (ordering=1).+ // group 1 keeps merged value=-6 (ordering=-6 < 6, so -6 is "first").+ // group 2 had no merged value (is_set=false), so update_batch value=6 wins.let expect: PrimitiveArray<Int64Type> =- Int64Array::from(vec![Some(1), Some(6), Some(6), None]);+ Int64Array::from(vec![Some(1), Some(-6), Some(6), None]);assert_eq!(eval_result, &expect);@@ -1680,7 +1677,7 @@ mod tests {group_acc.compute_size_of_orderings());- group_acc.merge_batch(&s, &Vec::from_iter(0..s[0].len()), None, 100)?;+ group_acc.merge_batch(&s, &Vec::from_iter(0..s[0].len()), 100)?;assert_eq!(group_acc.size_of_orderings,group_acc.compute_size_of_orderings()@@ -1753,12 +1750,7 @@ mod tests {];assert_eq!(state, expected_state);- group_acc.merge_batch(- &state,- &[0, 1, 2],- Some(&BooleanArray::from(vec![true, false, false])),- 3,- )?;+ group_acc.merge_batch(&state, &[0, 1, 2], 3)?;val_with_orderings.clear();val_with_orderings.push(Arc::new(Int64Array::from(vec![66, 6])));@@ -1769,6 +1761,10 @@ mod tests {let binding = group_acc.evaluate(EmitTo::All)?;let eval_result = binding.as_any().downcast_ref::<Int64Array>().unwrap();+ // group 0: merged value=1 (ordering=1, is_set=true), update not called.+ // group 1: merged value=-6 (ordering=-6, is_set=true); update ordering=66 > -6+ // → LAST_VALUE keeps the higher ordering, so group 1 becomes 66.+ // group 2: is_set=false after merge; update_batch sets it to 6.let expect: PrimitiveArray<Int64Type> =Int64Array::from(vec![Some(1), Some(66), Some(6), None]);diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rsindex e7e7d03937..4a0da10f51 100644--- a/datafusion/functions-aggregate/src/median.rs+++ b/datafusion/functions-aggregate/src/median.rs@@ -393,8 +393,6 @@ impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T&mut self,values: &[ArrayRef],group_indices: &[usize],- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "one argument to merge_batch");diff --git a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rsindex b56c2106e3..7a3c605d82 100644--- a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs+++ b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs@@ -309,11 +309,10 @@ impl GroupsAccumulator for MinMaxBytesAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// min/max are their own states (no transition needed)- self.update_batch(values, group_indices, opt_filter, total_num_groups)+ self.update_batch(values, group_indices, None, total_num_groups)}fn convert_to_state(diff --git a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs b/datafusion/functions-aggregate/src/min_max/min_max_struct.rsindex 7c94e7f573..10580ac18d 100644--- a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs+++ b/datafusion/functions-aggregate/src/min_max/min_max_struct.rs@@ -134,11 +134,10 @@ impl GroupsAccumulator for MinMaxStructAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// min/max are their own states (no transition needed)- self.update_batch(values, group_indices, opt_filter, total_num_groups)+ self.update_batch(values, group_indices, None, total_num_groups)}fn convert_to_state(diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rsindex 714988bde2..e8e6fd127e 100644--- a/datafusion/functions-aggregate/src/percentile_cont.rs+++ b/datafusion/functions-aggregate/src/percentile_cont.rs@@ -537,8 +537,6 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "one argument to merge_batch");diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rsindex 68e38a3b8d..f0482b23d1 100644--- a/datafusion/functions-aggregate/src/stddev.rs+++ b/datafusion/functions-aggregate/src/stddev.rs@@ -329,11 +329,10 @@ impl GroupsAccumulator for StddevGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&arrow::array::BooleanArray>,total_num_groups: usize,) -> Result<()> {self.variance- .merge_batch(values, group_indices, opt_filter, total_num_groups)+ .merge_batch(values, group_indices, total_num_groups)}fn evaluate(&mut self, emit_to: datafusion_expr::EmitTo) -> Result<ArrayRef> {diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rsindex f0757818af..6b0665f479 100644--- a/datafusion/functions-aggregate/src/string_agg.rs+++ b/datafusion/functions-aggregate/src/string_agg.rs@@ -413,11 +413,10 @@ impl GroupsAccumulator for StringAggGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {// State is always LargeUtf8, which update_batch already handles.- self.update_batch(values, group_indices, opt_filter, total_num_groups)+ self.update_batch(values, group_indices, None, total_num_groups)}fn convert_to_state(@@ -898,7 +897,7 @@ mod tests {// Simulate a second accumulator's state (LargeUtf8 partial strings)let partial_state: ArrayRef = Arc::new(LargeStringArray::from(vec!["c,d", "e"]));- acc.merge_batch(&[partial_state], &[0, 1], None, 2)?;+ acc.merge_batch(&[partial_state], &[0, 1], 2)?;let result = evaluate_groups(&mut acc, EmitTo::All);assert_eq!(diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rsindex d5fddf01f2..551fcfe120 100644--- a/datafusion/functions-aggregate/src/variance.rs+++ b/datafusion/functions-aggregate/src/variance.rs@@ -528,8 +528,6 @@ impl GroupsAccumulator for VarianceGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 3, "two arguments to merge_batch");@@ -673,8 +671,8 @@ mod tests {Arc::new(Float64Array::from(vec![1.0])),];let mut acc = VarianceGroupsAccumulator::new(StatsType::Sample);- acc.merge_batch(&state_1, &[0], None, 1)?;- acc.merge_batch(&state_2, &[0], None, 1)?;+ acc.merge_batch(&state_1, &[0], 1)?;+ acc.merge_batch(&state_2, &[0], 1)?;let result = acc.evaluate(EmitTo::All)?;let result = result.as_any().downcast_ref::<Float64Array>().unwrap();assert_eq!(result.len(), 1);diff --git a/datafusion/physical-plan/src/aggregates/hash_table.rs b/datafusion/physical-plan/src/aggregates/hash_table.rsindex 2e5702f750..e6b2fa22c1 100644--- a/datafusion/physical-plan/src/aggregates/hash_table.rs+++ b/datafusion/physical-plan/src/aggregates/hash_table.rs@@ -221,12 +221,8 @@ impl HashAggregateAccumulator {total_num_groups: usize,) -> Result<()> {debug_assert!(values.filter.is_none());- self.accumulator.merge_batch(- &values.arguments,- group_indices,- None,- total_num_groups,- )+ self.accumulator+ .merge_batch(&values.arguments, group_indices, total_num_groups)}fn evaluate_final(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rsindex 6f1f9f5768..d46faf9acc 100644--- a/datafusion/physical-plan/src/aggregates/row_hash.rs+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs@@ -923,7 +923,7 @@ impl GroupedHashAggregateStream {// if aggregation is over intermediate states,// use merge- acc.merge_batch(values, group_indices, None, total_num_groups)?;+ acc.merge_batch(values, group_indices, total_num_groups)?;}self.group_by_metrics.aggregation_time@@ -1111,17 +1111,31 @@ impl GroupedHashAggregateStream {// Prime each accumulator for the registered group count with no data.//// We build 1-row null arrays for each aggregate argument and pass them- // with an all-false filter. The filter ensures no row is accumulated- // into any group, which keeps every group in its "zero" initial state- // (NULL for SUM/AVG/MIN/MAX, 0 for COUNT).+ // with an all-false filter to update_batch. The filter ensures no row+ // is accumulated into any group, which keeps every group in its "zero"+ // initial state (NULL for SUM/AVG/MIN/MAX, 0 for COUNT).//// Using a 1-row batch rather than 0 rows is required to avoid a fast// path in `NullState::accumulate` that treats "0 nulls in a 0-row// array" as "all groups have been seen", which would cause SUM to// return 0 instead of NULL.//- // Argument types are inferred directly from the expression metadata so- // we never need to construct a full `RecordBatch`.+ // This path always runs in a Raw input mode, so `update_batch` (not+ // `merge_batch`) is the right entry point:+ //+ // - `has_grouping_set()` can only be true for the Partial / Single /+ // SinglePartitioned modes, whose `input_mode()` is `Raw`. The final+ // modes rebuild their group-by via `PhysicalGroupBy::as_final()`,+ // which clears `has_grouping_set`, so this method returns early for+ // them and never reaches here.+ //+ // Since every row is filtered out, the actual data content never+ // matters. The assert documents and guards the invariant above.+ debug_assert_eq!(+ self.mode.input_mode(),+ AggregateInputMode::Raw,+ "init_empty_grouping_sets must only run in a Raw input mode"+ );let total_groups = self.group_values.len();let null_args: Vec<Vec<ArrayRef>> = self.aggregate_arguments@@ -1137,11 +1151,7 @@ impl GroupedHashAggregateStream {.collect::<Result<Vec<_>>>()?;let false_filter = BooleanArray::from(vec![false]);for (acc, args) in self.accumulators.iter_mut().zip(null_args.iter()) {- if self.mode.input_mode() == AggregateInputMode::Raw {- acc.update_batch(args, &[0], Some(&false_filter), total_groups)?;- } else {- acc.merge_batch(args, &[0], Some(&false_filter), total_groups)?;- }+ acc.update_batch(args, &[0], Some(&false_filter), total_groups)?;}}diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rsindex 5f4d2c253a..6ca3c59309 100644--- a/datafusion/spark/src/function/aggregate/avg.rs+++ b/datafusion/spark/src/function/aggregate/avg.rs@@ -289,7 +289,6 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 2, "two arguments to merge_batch");@@ -464,7 +463,6 @@ mod tests {acc.merge_batch(&state,&[0, 0, 0],- None,1, // single group).unwrap();@@ -486,7 +484,7 @@ mod tests {Some(3.0),]))];let state = acc.convert_to_state(&input, None).unwrap();- acc.merge_batch(&state, &[0, 0, 0], None, 1).unwrap();+ acc.merge_batch(&state, &[0, 0, 0], 1).unwrap();let result = acc.evaluate(EmitTo::All).unwrap();let result = result.as_primitive::<Float64Type>();diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.sltindex eac901b2a3..2c05dd851e 100644--- a/datafusion/sqllogictest/test_files/grouping.slt+++ b/datafusion/sqllogictest/test_files/grouping.slt@@ -232,6 +232,12 @@ SELECT SUM(v1) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING----NULL+# grouping_sets_empty_input_avg: AVG returns NULL for the empty group+query R+SELECT AVG(v1) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())+----+NULL+# grouping_sets_empty_input_count: COUNT returns 0 for the empty group, not a missing rowquery ISELECT COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex 2ebb6952fe..abe99846c1 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -45,6 +45,58 @@ will now appear as `Decimal128(NULL,10,2)`.Query result values already used human-readable decimal formatting and areunchanged.+### `GroupsAccumulator::merge_batch` no longer takes `opt_filter`++The `opt_filter` argument has been removed from+`datafusion_expr_common::groups_accumulator::GroupsAccumulator::merge_batch`:++```diff+ fn merge_batch(+ &mut self,+ values: &[ArrayRef],+ group_indices: &[usize],+- opt_filter: Option<&BooleanArray>,+ total_num_groups: usize,+ ) -> Result<()>;+```++Aggregate `FILTER` clauses only apply to raw input rows during the partial+(update) phase, so by the time intermediate states are merged there is nothing+left to filter per row. In practice `opt_filter` was always `None` here, so+removing it makes the API self-explanatory and impossible to misuse.++**Who is affected:**++- Anyone with a custom `GroupsAccumulator` implementation.+- Anyone calling `merge_batch` directly.++**Migration guide:**++Drop the `opt_filter` argument from your `merge_batch` signature and from any+call sites:++```diff+ fn merge_batch(+ &mut self,+ values: &[ArrayRef],+ group_indices: &[usize],+- opt_filter: Option<&BooleanArray>,+ total_num_groups: usize,+ ) -> Result<()> {+ // ...+ }+```++```diff+- acc.merge_batch(values, group_indices, None, total_num_groups)?;++ acc.merge_batch(values, group_indices, total_num_groups)?;+```++If your implementation previously inspected `opt_filter` (for example asserting+it was `None`), that code can simply be deleted.++See [issue #22775](https://github.com/apache/datafusion/issues/22775) for details.+### `is_dynamic_physical_expr` is deprecated`datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr` is