Skip to content

Apache DataFusion SQL Query Engine

refactor: remove `opt_filter` in `GroupsAccumulator::merge_batch` (#22816)

Huaijinc14379b9b528
diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rs
index f1651dbf28..b990740159 100644
--- a/datafusion-examples/examples/udf/advanced_udaf.rs
+++ b/datafusion-examples/examples/udf/advanced_udaf.rs
@@ -268,7 +268,6 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&arrow::array::BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 2, "two arguments to merge_batch");
@@ -280,7 +279,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
self.null_state.accumulate(
group_indices,
partial_counts,
- opt_filter,
+ None,
total_num_groups,
|group_index, partial_count| {
self.counts[group_index] += partial_count;
@@ -292,7 +291,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {
self.null_state.accumulate(
group_indices,
partial_prods,
- opt_filter,
+ None,
total_num_groups,
|group_index, new_value: <Float64Type as ArrowPrimitiveType>::Native| {
let prod = &mut self.prods[group_index];
diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
index 7d22c5df70..b895cb9c7c 100644
--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs
+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
@@ -872,7 +872,6 @@ impl GroupsAccumulator for TestGroupsAccumulator {
&mut self,
_values: &[ArrayRef],
_group_indices: &[usize],
- _opt_filter: Option<&arrow::array::BooleanArray>,
_total_num_groups: usize,
) -> Result<()> {
Ok(())
diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rs
index da5da384c7..b021674cbe 100644
--- a/datafusion/expr-common/src/groups_accumulator.rs
+++ b/datafusion/expr-common/src/groups_accumulator.rs
@@ -183,12 +183,14 @@ pub trait GroupsAccumulator: Send + std::any::Any {
///
/// * `values`: arrays produced from previously calling `state` on other accumulators.
///
- /// Other arguments are the same as for [`Self::update_batch`].
+ /// Other arguments are the same as for [`Self::update_batch`], except that
+ /// there is no `opt_filter` — aggregate filters are applied during the
+ /// partial (update) phase, so by the time intermediate states are merged
+ /// no per-row filtering is needed.
fn merge_batch(
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()>;
diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rs
index 1600bef39d..272afdb6ab 100644
--- a/datafusion/ffi/src/udaf/groups_accumulator.rs
+++ b/datafusion/ffi/src/udaf/groups_accumulator.rs
@@ -64,7 +64,6 @@ pub struct FFI_GroupsAccumulator {
accumulator: &mut Self,
values: SVec<WrappedArray>,
group_indices: SVec<usize>,
- opt_filter: FFI_Option<WrappedArray>,
total_num_groups: usize,
) -> FFI_Result<()>,
@@ -195,21 +194,14 @@ unsafe extern "C" fn merge_batch_fn_wrapper(
accumulator: &mut FFI_GroupsAccumulator,
values: SVec<WrappedArray>,
group_indices: SVec<usize>,
- opt_filter: FFI_Option<WrappedArray>,
total_num_groups: usize,
) -> FFI_Result<()> {
unsafe {
let accumulator = accumulator.inner_mut();
let values = sresult_return!(process_values(values));
let group_indices: Vec<usize> = group_indices.into_iter().collect();
- let opt_filter = sresult_return!(process_opt_filter(opt_filter));
- sresult!(accumulator.merge_batch(
- &values,
- &group_indices,
- opt_filter.as_ref(),
- total_num_groups
- ))
+ sresult!(accumulator.merge_batch(&values, &group_indices, total_num_groups))
}
}
@@ -379,7 +371,6 @@ impl GroupsAccumulator for ForeignGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
unsafe {
@@ -388,20 +379,11 @@ impl GroupsAccumulator for ForeignGroupsAccumulator {
.map(WrappedArray::try_from)
.collect::<std::result::Result<Vec<_>, ArrowError>>()?;
let group_indices = group_indices.iter().cloned().collect();
- let opt_filter = opt_filter
- .map(|bool_array| to_ffi(&bool_array.to_data()))
- .transpose()?
- .map(|(array, schema)| WrappedArray {
- array,
- schema: WrappedSchema(schema),
- })
- .into();
df_result!((self.accumulator.merge_batch)(
&mut self.accumulator,
values.into_iter().collect(),
group_indices,
- opt_filter,
total_num_groups
))
}
@@ -517,8 +499,7 @@ mod tests {
let second_states =
vec![make_array(create_array!(Boolean, vec![false]).to_data())];
- let opt_filter = create_array!(Boolean, vec![true]);
- foreign_accum.merge_batch(&second_states, &[0], Some(opt_filter.as_ref()), 1)?;
+ foreign_accum.merge_batch(&second_states, &[0], 1)?;
let groups_bool = foreign_accum.evaluate(EmitTo::All)?;
assert_eq!(groups_bool.len(), 1);
assert_eq!(
diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs
index d370d59c90..60fe0388c4 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs
@@ -160,7 +160,6 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> datafusion_common::Result<()> {
debug_assert_eq!(values.len(), 1);
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
index ad2a21bb47..b412b4ffe0 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
@@ -375,13 +375,12 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
self.invoke_per_accumulator(
values,
group_indices,
- opt_filter,
+ None,
total_num_groups,
|accumulator, values_to_accumulate| {
accumulator.merge_batch(values_to_accumulate)?;
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs
index d1d8924a2c..afb1dec24a 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs
@@ -132,11 +132,10 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// update / merge are the same
- self.update_batch(values, group_indices, opt_filter, total_num_groups)
+ self.update_batch(values, group_indices, None, total_num_groups)
}
fn size(&self) -> usize {
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs
index a81b89e1e4..474899d8f3 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs
@@ -131,11 +131,10 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// update / merge are the same
- self.update_batch(values, group_indices, opt_filter, total_num_groups)
+ self.update_batch(values, group_indices, None, total_num_groups)
}
/// Converts an input batch directly to a state batch
diff --git a/datafusion/functions-aggregate/benches/first_last.rs b/datafusion/functions-aggregate/benches/first_last.rs
index 1d18e1c7dc..8f28e126a4 100644
--- a/datafusion/functions-aggregate/benches/first_last.rs
+++ b/datafusion/functions-aggregate/benches/first_last.rs
@@ -235,7 +235,6 @@ fn merge_bench(
Arc::clone(&is_set),
],
&group_indices,
- opt_filter,
num_groups,
)
.unwrap(),
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index 38b902964f..0c5a438454 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -705,15 +705,8 @@ impl<H: HllValueHasher> GroupsAccumulator for HllGroupsAccumulator<H> {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
- assert!(
- opt_filter.is_none(),
- "aggregate filter should be applied in partial stage, there should be no filter in final stage"
- );
-
self.ensure_groups(total_num_groups);
let states = downcast_value!(values[0], BinaryArray);
let mut delta: isize = 0;
diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs
index 8ed3fbf8c3..1dd111f918 100644
--- a/datafusion/functions-aggregate/src/array_agg.rs
+++ b/datafusion/functions-aggregate/src/array_agg.rs
@@ -732,7 +732,6 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 1, "one argument to merge_batch");
@@ -2019,7 +2018,7 @@ mod tests {
// Merge acc2's state into acc1
let state = acc2.state(EmitTo::All)?;
- acc1.merge_batch(&state, &[0, 1], None, 2)?;
+ acc1.merge_batch(&state, &[0, 1], 2)?;
// Another update_batch on acc1 after the merge
let values: ArrayRef = Arc::new(Int32Array::from(vec![5, 6]));
@@ -2088,7 +2087,7 @@ mod tests {
// Feed state into a new accumulator via merge_batch
let mut acc2 = ArrayAggGroupsAccumulator::new(DataType::Int32, false);
- acc2.merge_batch(&state, &[0, 0, 1], None, 2)?;
+ acc2.merge_batch(&state, &[0, 0, 1], 2)?;
// Group 0 received rows 0 ([1]) and 1 ([NULL]) → [1, NULL]
let vals = eval_i32_lists(&mut acc2, EmitTo::All)?;
@@ -2118,7 +2117,7 @@ mod tests {
// Feed state into a new accumulator via merge_batch
let mut acc2 = ArrayAggGroupsAccumulator::new(DataType::Int32, true);
- acc2.merge_batch(&state, &[0, 0, 1, 1], None, 2)?;
+ acc2.merge_batch(&state, &[0, 0, 1, 1], 2)?;
// Group 0: received [1] and null (skipped) → [1]
let vals = eval_i32_lists(&mut acc2, EmitTo::All)?;
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs
index ddeb9b0870..06c7694634 100644
--- a/datafusion/functions-aggregate/src/average.rs
+++ b/datafusion/functions-aggregate/src/average.rs
@@ -908,7 +908,6 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 2, "two arguments to merge_batch");
@@ -920,7 +919,7 @@ where
self.null_state.accumulate(
group_indices,
partial_counts,
- opt_filter,
+ None,
total_num_groups,
|group_index, partial_count| {
// SAFETY: group_index is guaranteed to be in bounds
@@ -934,7 +933,7 @@ where
self.null_state.accumulate(
group_indices,
partial_sums,
- opt_filter,
+ None,
total_num_groups,
|group_index, new_value: <T as ArrowPrimitiveType>::Native| {
// SAFETY: group_index is guaranteed to be in bounds
diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rs
index 5a95cfe832..7fcf4bb61f 100644
--- a/datafusion/functions-aggregate/src/correlation.rs
+++ b/datafusion/functions-aggregate/src/correlation.rs
@@ -493,7 +493,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// Resize vectors to accommodate total number of groups
@@ -512,11 +511,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {
let partial_sum_xx = values[4].as_primitive::<Float64Type>();
let partial_sum_yy = values[5].as_primitive::<Float64Type>();
- assert!(
- opt_filter.is_none(),
- "aggregate filter should be applied in partial stage, there should be no filter in final stage"
- );
-
accumulate_correlation_states(
group_indices,
(
diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs
index eab36d4951..f0ce8c82a1 100644
--- a/datafusion/functions-aggregate/src/count.rs
+++ b/datafusion/functions-aggregate/src/count.rs
@@ -665,8 +665,6 @@ impl GroupsAccumulator for CountGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 1, "one argument to merge_batch");
diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs
index 1935f29c4c..cecb277cb8 100644
--- a/datafusion/functions-aggregate/src/first_last.rs
+++ b/datafusion/functions-aggregate/src/first_last.rs
@@ -671,7 +671,6 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
self.resize_states(total_num_groups);
@@ -690,7 +689,7 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S
let groups = self.get_filtered_extreme_of_each_group(
&val_and_order_cols[1..],
group_indices,
- opt_filter,
+ None,
vals,
Some(is_set_arr),
)?;
@@ -1587,12 +1586,7 @@ mod tests {
group_acc.compute_size_of_orderings()
);
- group_acc.merge_batch(
- &state,
- &[0, 1, 2],
- Some(&BooleanArray::from(vec![true, false, false])),
- 3,
- )?;
+ group_acc.merge_batch(&state, &[0, 1, 2], 3)?;
assert_eq!(
group_acc.size_of_orderings,
@@ -1608,8 +1602,11 @@ mod tests {
let binding = group_acc.evaluate(EmitTo::All)?;
let eval_result = binding.as_any().downcast_ref::<Int64Array>().unwrap();
+ // group 0 keeps merged value=1 (ordering=1).
+ // group 1 keeps merged value=-6 (ordering=-6 < 6, so -6 is "first").
+ // group 2 had no merged value (is_set=false), so update_batch value=6 wins.
let expect: PrimitiveArray<Int64Type> =
- Int64Array::from(vec![Some(1), Some(6), Some(6), None]);
+ Int64Array::from(vec![Some(1), Some(-6), Some(6), None]);
assert_eq!(eval_result, &expect);
@@ -1680,7 +1677,7 @@ mod tests {
group_acc.compute_size_of_orderings()
);
- group_acc.merge_batch(&s, &Vec::from_iter(0..s[0].len()), None, 100)?;
+ group_acc.merge_batch(&s, &Vec::from_iter(0..s[0].len()), 100)?;
assert_eq!(
group_acc.size_of_orderings,
group_acc.compute_size_of_orderings()
@@ -1753,12 +1750,7 @@ mod tests {
];
assert_eq!(state, expected_state);
- group_acc.merge_batch(
- &state,
- &[0, 1, 2],
- Some(&BooleanArray::from(vec![true, false, false])),
- 3,
- )?;
+ group_acc.merge_batch(&state, &[0, 1, 2], 3)?;
val_with_orderings.clear();
val_with_orderings.push(Arc::new(Int64Array::from(vec![66, 6])));
@@ -1769,6 +1761,10 @@ mod tests {
let binding = group_acc.evaluate(EmitTo::All)?;
let eval_result = binding.as_any().downcast_ref::<Int64Array>().unwrap();
+ // group 0: merged value=1 (ordering=1, is_set=true), update not called.
+ // group 1: merged value=-6 (ordering=-6, is_set=true); update ordering=66 > -6
+ // → LAST_VALUE keeps the higher ordering, so group 1 becomes 66.
+ // group 2: is_set=false after merge; update_batch sets it to 6.
let expect: PrimitiveArray<Int64Type> =
Int64Array::from(vec![Some(1), Some(66), Some(6), None]);
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs
index e7e7d03937..4a0da10f51 100644
--- a/datafusion/functions-aggregate/src/median.rs
+++ b/datafusion/functions-aggregate/src/median.rs
@@ -393,8 +393,6 @@ impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 1, "one argument to merge_batch");
diff --git a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs
index b56c2106e3..7a3c605d82 100644
--- a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs
+++ b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs
@@ -309,11 +309,10 @@ impl GroupsAccumulator for MinMaxBytesAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// min/max are their own states (no transition needed)
- self.update_batch(values, group_indices, opt_filter, total_num_groups)
+ self.update_batch(values, group_indices, None, total_num_groups)
}
fn convert_to_state(
diff --git a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs b/datafusion/functions-aggregate/src/min_max/min_max_struct.rs
index 7c94e7f573..10580ac18d 100644
--- a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs
+++ b/datafusion/functions-aggregate/src/min_max/min_max_struct.rs
@@ -134,11 +134,10 @@ impl GroupsAccumulator for MinMaxStructAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// min/max are their own states (no transition needed)
- self.update_batch(values, group_indices, opt_filter, total_num_groups)
+ self.update_batch(values, group_indices, None, total_num_groups)
}
fn convert_to_state(
diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs
index 714988bde2..e8e6fd127e 100644
--- a/datafusion/functions-aggregate/src/percentile_cont.rs
+++ b/datafusion/functions-aggregate/src/percentile_cont.rs
@@ -537,8 +537,6 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 1, "one argument to merge_batch");
diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rs
index 68e38a3b8d..f0482b23d1 100644
--- a/datafusion/functions-aggregate/src/stddev.rs
+++ b/datafusion/functions-aggregate/src/stddev.rs
@@ -329,11 +329,10 @@ impl GroupsAccumulator for StddevGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&arrow::array::BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
self.variance
- .merge_batch(values, group_indices, opt_filter, total_num_groups)
+ .merge_batch(values, group_indices, total_num_groups)
}
fn evaluate(&mut self, emit_to: datafusion_expr::EmitTo) -> Result<ArrayRef> {
diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs
index f0757818af..6b0665f479 100644
--- a/datafusion/functions-aggregate/src/string_agg.rs
+++ b/datafusion/functions-aggregate/src/string_agg.rs
@@ -413,11 +413,10 @@ impl GroupsAccumulator for StringAggGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
// State is always LargeUtf8, which update_batch already handles.
- self.update_batch(values, group_indices, opt_filter, total_num_groups)
+ self.update_batch(values, group_indices, None, total_num_groups)
}
fn convert_to_state(
@@ -898,7 +897,7 @@ mod tests {
// Simulate a second accumulator's state (LargeUtf8 partial strings)
let partial_state: ArrayRef = Arc::new(LargeStringArray::from(vec!["c,d", "e"]));
- acc.merge_batch(&[partial_state], &[0, 1], None, 2)?;
+ acc.merge_batch(&[partial_state], &[0, 1], 2)?;
let result = evaluate_groups(&mut acc, EmitTo::All);
assert_eq!(
diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs
index d5fddf01f2..551fcfe120 100644
--- a/datafusion/functions-aggregate/src/variance.rs
+++ b/datafusion/functions-aggregate/src/variance.rs
@@ -528,8 +528,6 @@ impl GroupsAccumulator for VarianceGroupsAccumulator {
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- // Since aggregate filter should be applied in partial stage, in final stage there should be no filter
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 3, "two arguments to merge_batch");
@@ -673,8 +671,8 @@ mod tests {
Arc::new(Float64Array::from(vec![1.0])),
];
let mut acc = VarianceGroupsAccumulator::new(StatsType::Sample);
- acc.merge_batch(&state_1, &[0], None, 1)?;
- acc.merge_batch(&state_2, &[0], None, 1)?;
+ acc.merge_batch(&state_1, &[0], 1)?;
+ acc.merge_batch(&state_2, &[0], 1)?;
let result = acc.evaluate(EmitTo::All)?;
let result = result.as_any().downcast_ref::<Float64Array>().unwrap();
assert_eq!(result.len(), 1);
diff --git a/datafusion/physical-plan/src/aggregates/hash_table.rs b/datafusion/physical-plan/src/aggregates/hash_table.rs
index 2e5702f750..e6b2fa22c1 100644
--- a/datafusion/physical-plan/src/aggregates/hash_table.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_table.rs
@@ -221,12 +221,8 @@ impl HashAggregateAccumulator {
total_num_groups: usize,
) -> Result<()> {
debug_assert!(values.filter.is_none());
- self.accumulator.merge_batch(
- &values.arguments,
- group_indices,
- None,
- total_num_groups,
- )
+ self.accumulator
+ .merge_batch(&values.arguments, group_indices, total_num_groups)
}
fn evaluate_final(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rs
index 6f1f9f5768..d46faf9acc 100644
--- a/datafusion/physical-plan/src/aggregates/row_hash.rs
+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs
@@ -923,7 +923,7 @@ impl GroupedHashAggregateStream {
// if aggregation is over intermediate states,
// use merge
- acc.merge_batch(values, group_indices, None, total_num_groups)?;
+ acc.merge_batch(values, group_indices, total_num_groups)?;
}
self.group_by_metrics
.aggregation_time
@@ -1111,17 +1111,31 @@ impl GroupedHashAggregateStream {
// Prime each accumulator for the registered group count with no data.
//
// We build 1-row null arrays for each aggregate argument and pass them
- // with an all-false filter. The filter ensures no row is accumulated
- // into any group, which keeps every group in its "zero" initial state
- // (NULL for SUM/AVG/MIN/MAX, 0 for COUNT).
+ // with an all-false filter to update_batch. The filter ensures no row
+ // is accumulated into any group, which keeps every group in its "zero"
+ // initial state (NULL for SUM/AVG/MIN/MAX, 0 for COUNT).
//
// Using a 1-row batch rather than 0 rows is required to avoid a fast
// path in `NullState::accumulate` that treats "0 nulls in a 0-row
// array" as "all groups have been seen", which would cause SUM to
// return 0 instead of NULL.
//
- // Argument types are inferred directly from the expression metadata so
- // we never need to construct a full `RecordBatch`.
+ // This path always runs in a Raw input mode, so `update_batch` (not
+ // `merge_batch`) is the right entry point:
+ //
+ // - `has_grouping_set()` can only be true for the Partial / Single /
+ // SinglePartitioned modes, whose `input_mode()` is `Raw`. The final
+ // modes rebuild their group-by via `PhysicalGroupBy::as_final()`,
+ // which clears `has_grouping_set`, so this method returns early for
+ // them and never reaches here.
+ //
+ // Since every row is filtered out, the actual data content never
+ // matters. The assert documents and guards the invariant above.
+ debug_assert_eq!(
+ self.mode.input_mode(),
+ AggregateInputMode::Raw,
+ "init_empty_grouping_sets must only run in a Raw input mode"
+ );
let total_groups = self.group_values.len();
let null_args: Vec<Vec<ArrayRef>> = self
.aggregate_arguments
@@ -1137,11 +1151,7 @@ impl GroupedHashAggregateStream {
.collect::<Result<Vec<_>>>()?;
let false_filter = BooleanArray::from(vec![false]);
for (acc, args) in self.accumulators.iter_mut().zip(null_args.iter()) {
- if self.mode.input_mode() == AggregateInputMode::Raw {
- acc.update_batch(args, &[0], Some(&false_filter), total_groups)?;
- } else {
- acc.merge_batch(args, &[0], Some(&false_filter), total_groups)?;
- }
+ acc.update_batch(args, &[0], Some(&false_filter), total_groups)?;
}
}
diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rs
index 5f4d2c253a..6ca3c59309 100644
--- a/datafusion/spark/src/function/aggregate/avg.rs
+++ b/datafusion/spark/src/function/aggregate/avg.rs
@@ -289,7 +289,6 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 2, "two arguments to merge_batch");
@@ -464,7 +463,6 @@ mod tests {
acc.merge_batch(
&state,
&[0, 0, 0],
- None,
1, // single group
)
.unwrap();
@@ -486,7 +484,7 @@ mod tests {
Some(3.0),
]))];
let state = acc.convert_to_state(&input, None).unwrap();
- acc.merge_batch(&state, &[0, 0, 0], None, 1).unwrap();
+ acc.merge_batch(&state, &[0, 0, 0], 1).unwrap();
let result = acc.evaluate(EmitTo::All).unwrap();
let result = result.as_primitive::<Float64Type>();
diff --git a/datafusion/sqllogictest/test_files/grouping.slt b/datafusion/sqllogictest/test_files/grouping.slt
index eac901b2a3..2c05dd851e 100644
--- a/datafusion/sqllogictest/test_files/grouping.slt
+++ b/datafusion/sqllogictest/test_files/grouping.slt
@@ -232,6 +232,12 @@ SELECT SUM(v1) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING
----
NULL
+# grouping_sets_empty_input_avg: AVG returns NULL for the empty group
+query R
+SELECT AVG(v1) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())
+----
+NULL
+
# grouping_sets_empty_input_count: COUNT returns 0 for the empty group, not a missing row
query I
SELECT COUNT(*) FROM generate_series(10) AS t1(v1) WHERE false GROUP BY GROUPING SETS(())
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index 2ebb6952fe..abe99846c1 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -45,6 +45,58 @@ will now appear as `Decimal128(NULL,10,2)`.
Query result values already used human-readable decimal formatting and are
unchanged.
+### `GroupsAccumulator::merge_batch` no longer takes `opt_filter`
+
+The `opt_filter` argument has been removed from
+`datafusion_expr_common::groups_accumulator::GroupsAccumulator::merge_batch`:
+
+```diff
+ fn merge_batch(
+ &mut self,
+ values: &[ArrayRef],
+ group_indices: &[usize],
+- opt_filter: Option<&BooleanArray>,
+ total_num_groups: usize,
+ ) -> Result<()>;
+```
+
+Aggregate `FILTER` clauses only apply to raw input rows during the partial
+(update) phase, so by the time intermediate states are merged there is nothing
+left to filter per row. In practice `opt_filter` was always `None` here, so
+removing it makes the API self-explanatory and impossible to misuse.
+
+**Who is affected:**
+
+- Anyone with a custom `GroupsAccumulator` implementation.
+- Anyone calling `merge_batch` directly.
+
+**Migration guide:**
+
+Drop the `opt_filter` argument from your `merge_batch` signature and from any
+call sites:
+
+```diff
+ fn merge_batch(
+ &mut self,
+ values: &[ArrayRef],
+ group_indices: &[usize],
+- opt_filter: Option<&BooleanArray>,
+ total_num_groups: usize,
+ ) -> Result<()> {
+ // ...
+ }
+```
+
+```diff
+- acc.merge_batch(values, group_indices, None, total_num_groups)?;
++ acc.merge_batch(values, group_indices, total_num_groups)?;
+```
+
+If your implementation previously inspected `opt_filter` (for example asserting
+it was `None`), that code can simply be deleted.
+
+See [issue #22775](https://github.com/apache/datafusion/issues/22775) for details.
+
### `is_dynamic_physical_expr` is deprecated
`datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr` is