Apache DataFusion SQL Query Engine
perf: filter only aggregate argument columns (#27119)
xudong.w34eb6dc5bf05
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_argument.rs b/datafusion/physical-plan/src/aggregates/aggregate_argument.rsindex fde252c231..1b462ee071 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_argument.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_argument.rs@@ -7,10 +7,16 @@use std::sync::Arc;use crate::PhysicalExpr;-use arrow::array::{ArrayRef, BooleanArray};-use arrow::record_batch::RecordBatch;-use datafusion_common::{Result, ScalarValue};+use arrow::array::{Array, ArrayRef, BooleanArray, NullArray, new_null_array};+use arrow::compute::FilterBuilder;+use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};+use arrow::record_batch::{RecordBatch, RecordBatchOptions};+use datafusion_common::cast::as_boolean_array;+use datafusion_common::{Result, ScalarValue, internal_err};use datafusion_physical_expr::expressions::Literal;+use datafusion_physical_expr::utils::collect_columns;+use datafusion_physical_expr_common::utils::scatter;+use itertools::Itertools;/// Largest literal array, in bytes, that an [`AggregateArgument`] keeps for/// later batches.@@ -35,16 +41,42 @@ const MAX_KEPT_LITERAL_ARRAY_BYTES: usize = 1024 * 1024;/// [`ColumnarValue::Scalar`]: datafusion_expr::ColumnarValue::Scalarpub(in crate::aggregates) struct AggregateArgument {expr: Arc<dyn PhysicalExpr>,+ /// The argument's type in the input schema, also used for rejected rows.+ data_type: DataType,/// Set when `expr` is a [`Literal`].literal: Option<LiteralArray>,}impl AggregateArgument {- pub(in crate::aggregates) fn new(expr: Arc<dyn PhysicalExpr>) -> Self {+ pub(in crate::aggregates) fn try_new(+ expr: Arc<dyn PhysicalExpr>,+ input_schema: &Schema,+ ) -> Result<Self> {+ let data_type = expr.data_type(input_schema)?;let literal = expr.downcast_ref::<Literal>().map(|literal| LiteralArray::new(literal.value().clone()));- Self { expr, literal }+ Ok(Self {+ expr,+ data_type,+ literal,+ })+ }++ /// Keeps the expression and its type without retaining a literal array.+ pub(in crate::aggregates) fn empty_like(&self) -> Self {+ Self {+ expr: Arc::clone(&self.expr),+ data_type: self.data_type.clone(),+ literal: self+ .literal+ .as_ref()+ .map(|literal| LiteralArray::new(literal.value.clone())),+ }+ }++ pub(in crate::aggregates) fn null_array(&self, num_rows: usize) -> ArrayRef {+ new_null_array(&self.data_type, num_rows)}pub(in crate::aggregates) fn expr(&self) -> &Arc<dyn PhysicalExpr> {@@ -68,6 +100,10 @@ impl AggregateArgument {/// Evaluates the argument for the rows of `batch` where `selection` is/// true, returning one value per row of `batch`.///+ /// `selected` holds those rows, as returned by [`AggregateFilter::select`].+ /// Like [`PhysicalExpr::evaluate_selection`], the argument is evaluated+ /// only against them, so rejected rows cannot raise errors.+ ////// Rows where `selection` is not true may hold any value, because the/// selection is also passed to [`GroupsAccumulator::convert_to_state`],/// which ignores them.@@ -77,16 +113,160 @@ impl AggregateArgument {&mut self,batch: &RecordBatch,selection: &BooleanArray,+ selected: &RecordBatch,) -> Result<ArrayRef> {+ let num_rows = batch.num_rows();+ if selected.num_rows() == num_rows {+ return self.evaluate(batch);+ }+ if selected.num_rows() == 0 {+ // A fallible expression may fail even for an empty batch, so it+ // is not evaluated.+ return Ok(self.null_array(num_rows));+ }match &mut self.literal {- Some(literal) if selection.has_true() => {- literal.array_of_size(batch.num_rows())- }- _ => self- .expr- .evaluate_selection(batch, selection)?- .into_array_of_size(batch.num_rows()),+ Some(literal) => literal.array_of_size(num_rows),+ None => scatter(selection, self.evaluate(selected)?.as_ref()),+ }+ }+}++/// The `FILTER` clause of one aggregate function, such as `x > 0` in+/// `SUM(y) FILTER (WHERE x > 0)`.+///+/// The arguments of the aggregate are evaluated only for the rows the filter+/// selects, so those rows are copied out of each input batch first: evaluating+/// the arguments for every row could raise errors for rejected rows, such as+/// division by zero in `SUM(1 / x) FILTER (WHERE x <> 0)`.+///+/// Only the input columns that the arguments read are copied. A query may have+/// many filtered aggregates over a wide input, and copying every column for+/// every aggregate would dominate the cost of the aggregation.+pub(in crate::aggregates) struct AggregateFilter {+ predicate: Arc<dyn PhysicalExpr>,+ /// Sorted indices of the input columns that the arguments read.+ argument_columns: Vec<usize>,+ /// The input schema last passed to [`Self::select`], and the schema of the+ /// batches it returns for that input.+ selected_schema: Option<(SchemaRef, SchemaRef)>,+}++impl AggregateFilter {+ pub(in crate::aggregates) fn new(+ predicate: Arc<dyn PhysicalExpr>,+ arguments: &[AggregateArgument],+ ) -> Self {+ let argument_columns = arguments+ .iter()+ .flat_map(|argument| collect_columns(argument.expr()))+ .map(|column| column.index())+ .sorted_unstable()+ .dedup()+ .collect();+ Self {+ predicate,+ argument_columns,+ selected_schema: None,+ }+ }++ pub(in crate::aggregates) fn predicate(&self) -> &Arc<dyn PhysicalExpr> {+ &self.predicate+ }++ /// Evaluates the filter predicate for every row of `batch`.+ pub(in crate::aggregates) fn evaluate(+ &self,+ batch: &RecordBatch,+ ) -> Result<BooleanArray> {+ let selection = self+ .predicate+ .evaluate(batch)?+ .into_array(batch.num_rows())?;+ Ok(as_boolean_array(&selection)?.clone())+ }++ /// Returns the rows of `batch` where `selection` is true, for evaluating+ /// the arguments of the aggregate.+ ///+ /// Only the columns that the arguments read are copied. The other columns+ /// are replaced by [`NullArray`]s, which hold no data, so the column+ /// indices in the arguments remain valid. If every row is selected, the+ /// input batch is shared without changing its schema or columns.+ pub(in crate::aggregates) fn select(+ &mut self,+ batch: &RecordBatch,+ selection: &BooleanArray,+ ) -> Result<RecordBatch> {+ // Nulls reject their rows, but `has_false` does not count them.+ if selection.null_count() == 0 && !selection.has_false() {+ return Ok(batch.clone());+ }+ let schema = batch.schema_ref();+ // As in `filter_record_batch`, optimizing the filter pays off only+ // when it is applied to several arrays.+ let optimize = match self.argument_columns.as_slice() {+ [] => false,+ [index] => schema.fields().get(*index).is_some_and(|field| {+ FilterBuilder::is_optimize_beneficial(field.data_type())+ }),+ _ => true,+ };+ let mut builder = FilterBuilder::new(selection);+ if optimize {+ builder = builder.optimize();+ }+ let predicate = builder.build();++ let num_rows = predicate.count();+ // Every column the arguments do not read shares one placeholder.+ let placeholder: ArrayRef = Arc::new(NullArray::new(num_rows));+ let mut columns = vec![placeholder; batch.num_columns()];+ for &index in &self.argument_columns {+ let Some(column) = batch.columns().get(index) else {+ return internal_err!(+ "Aggregate argument references column {index}, but the input has {} columns",+ batch.num_columns()+ );+ };+ columns[index] = predicate.filter(column.as_ref())?;+ }++ let options = RecordBatchOptions::new().with_row_count(Some(num_rows));+ Ok(RecordBatch::try_new_with_options(+ self.selected_schema(schema),+ columns,+ &options,+ )?)+ }++ /// Returns `schema` with the fields of columns that the arguments do not+ /// read changed to [`DataType::Null`].+ fn selected_schema(&mut self, schema: &SchemaRef) -> SchemaRef {+ if let Some((input, selected)) = &self.selected_schema+ && input == schema+ {+ return Arc::clone(selected);}++ // Schema::project would remove fields and change the column indices+ // used by the existing argument expressions.+ let fields = schema+ .fields()+ .iter()+ .enumerate()+ .map(|(index, field)| {+ if self.argument_columns.binary_search(&index).is_ok() {+ Arc::clone(field)+ } else {+ Arc::new(Field::new(field.name(), DataType::Null, true))+ }+ })+ .collect::<Fields>();+ let selected =+ Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone()));+ self.selected_schema = Some((Arc::clone(schema), Arc::clone(&selected)));+ selected}}@@ -122,9 +302,11 @@ impl LiteralArray {mod tests {use super::*;- use arrow::array::{AsArray, Int32Array};- use arrow::datatypes::{DataType, Field, Int32Type, Schema};- use datafusion_physical_expr::expressions::{col, lit};+ use arrow::array::{AsArray, Int32Array, StringArray};+ use arrow::compute::filter_record_batch;+ use arrow::datatypes::Int32Type;+ use datafusion_expr::Operator;+ use datafusion_physical_expr::expressions::{binary, col, lit};fn batch(num_rows: usize) -> RecordBatch {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));@@ -138,7 +320,7 @@ mod tests {#[test]fn literal_array_is_reused_across_batches() -> Result<()> {- let mut argument = AggregateArgument::new(lit(1i32));+ let mut argument = AggregateArgument::try_new(lit(1i32), batch(0).schema_ref())?;let first = argument.evaluate(&batch(4))?;let smaller = argument.evaluate(&batch(3))?;@@ -163,7 +345,8 @@ mod tests {#[test]fn large_literal_array_is_not_kept() -> Result<()> {- let mut argument = AggregateArgument::new(lit("x".repeat(1024)));+ let mut argument =+ AggregateArgument::try_new(lit("x".repeat(1024)), batch(0).schema_ref())?;let first = argument.evaluate(&batch(2048))?;let second = argument.evaluate(&batch(2048))?;@@ -178,7 +361,7 @@ mod tests {#[test]fn non_literal_is_evaluated_for_every_batch() -> Result<()> {let schema = batch(0).schema();- let mut argument = AggregateArgument::new(col("a", &schema)?);+ let mut argument = AggregateArgument::try_new(col("a", &schema)?, &schema)?;let first = argument.evaluate(&batch(3))?;assert_eq!(@@ -194,28 +377,71 @@ mod tests {}#[test]- fn literal_selection_matches_evaluate_selection() -> Result<()> {- let batch = batch(3);- let expr = lit(7i32);- let mut argument = AggregateArgument::new(Arc::clone(&expr));-- for selection in [- BooleanArray::from(vec![true, true, true]),- BooleanArray::from(vec![false, true, false]),- BooleanArray::from(vec![Some(false), None, Some(true)]),- BooleanArray::from(vec![false, false, false]),- BooleanArray::from(vec![None, Some(false), None]),- ] {- let expected = expr- .evaluate_selection(&batch, &selection)?- .into_array(batch.num_rows())?;- let actual = argument.evaluate_selection(&batch, &selection)?;- assert_eq!(- actual.as_ref(),- expected.as_ref(),- "selection {selection:?}"- );+ fn filter_selects_only_argument_columns() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Utf8, true),+ Field::new("c", DataType::Int32, false),+ ]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3, 4])),+ Arc::new(StringArray::from(vec!["w", "x", "y", "z"])),+ Arc::new(Int32Array::from(vec![10, 20, 30, 40])),+ ],+ )?;+ let selection =+ BooleanArray::from(vec![Some(true), Some(false), None, Some(true)]);+ let expected = filter_record_batch(&batch, &selection)?;++ // `c + a` reads columns 0 and 2, so column 1 is not copied.+ let expr = binary(+ col("c", &schema)?,+ Operator::Plus,+ col("a", &schema)?,+ &schema,+ )?;+ let mut argument = AggregateArgument::try_new(expr, &schema)?;+ let mut filter = AggregateFilter::new(lit(true), std::slice::from_ref(&argument));+ let selected = filter.select(&batch, &selection)?;+ assert_eq!(selected.num_rows(), 2);+ assert_eq!(selected.column(0), expected.column(0));+ assert_eq!(selected.column(2), expected.column(2));+ assert_eq!(selected.column(1).data_type(), &DataType::Null);+ assert_eq!(selected.column(1).len(), 2);+ assert_eq!(selected.schema().field(1).name(), "b");+ assert_eq!(+ argument.evaluate(&selected)?.as_ref(),+ argument.evaluate(&expected)?.as_ref()+ );++ // The schema of the selected rows is reused for later batches.+ let next =+ filter.select(&batch, &BooleanArray::from(vec![true, true, false, true]))?;+ assert_eq!(next.num_rows(), 3);+ assert!(Arc::ptr_eq(selected.schema_ref(), next.schema_ref()));++ // Selecting every row shares the original arrays and schema.+ let all = filter.select(&batch, &BooleanArray::from(vec![true; 4]))?;+ assert!(Arc::ptr_eq(batch.schema_ref(), all.schema_ref()));+ for (input, selected) in batch.columns().iter().zip(all.columns()) {+ assert!(Arc::ptr_eq(input, selected));}++ // `COUNT(*)` reads no columns.+ let mut filter = AggregateFilter::new(+ lit(true),+ &[AggregateArgument::try_new(lit(1), &schema)?],+ );+ let selected = filter.select(&batch, &selection)?;+ assert_eq!(selected.num_rows(), 2);+ assert!(+ selected+ .columns()+ .iter()+ .all(|column| column.data_type() == &DataType::Null)+ );Ok(())}}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex dcee4fc334..fa7a134733 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -7,10 +7,8 @@ use std::collections::HashMap;use std::marker::PhantomData;use std::sync::Arc;-use arrow::array::{- Array, ArrayRef, AsArray, BooleanArray, new_empty_array, new_null_array,-};-use arrow::compute::{filter_record_batch, prep_null_mask_filter};+use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, new_null_array};+use arrow::compute::prep_null_mask_filter;use arrow::datatypes::SchemaRef;use arrow::downcast_dictionary_array;use arrow::record_batch::RecordBatch;@@ -23,7 +21,7 @@ use datafusion_physical_expr::aggregate::AggregateFunctionExpr;use log::debug;use crate::PhysicalExpr;-use crate::aggregates::aggregate_argument::AggregateArgument;+use crate::aggregates::aggregate_argument::{AggregateArgument, AggregateFilter};use crate::aggregates::group_values::{AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,GroupByMetrics, GroupValues, new_group_values,@@ -159,6 +157,10 @@ impl<AggrMode> AggregateHashTable<AggrMode> {.map(|(((agg_expr, arguments), filter), submetrics)| {let accumulator =create_group_accumulator(agg_expr, Arc::clone(submetrics))?;+ let arguments = arguments+ .into_iter()+ .map(|expr| AggregateArgument::try_new(expr, &input_schema))+ .collect::<Result<_>>()?;Ok(HashAggregateAccumulator::new(Arc::clone(agg_expr),arguments,@@ -482,13 +484,11 @@ impl<AggrMode> AggregateHashTable<AggrMode> {let values = state.accumulators.iter()- .map(|acc| {- Ok(CompactedAccumulatorArgs {- arguments: acc.null_arguments(&self.input_schema, 0)?,- selection: None,- })+ .map(|acc| CompactedAccumulatorArgs {+ arguments: acc.null_arguments(0),+ selection: None,})- .collect::<Result<Vec<_>>>()?;+ .collect::<Vec<_>>();group_by_metrics.time_aggregation(|| {for (idx, (acc, values)) instate.accumulators.iter_mut().zip(values.iter()).enumerate()@@ -519,10 +519,10 @@ pub(super) struct HashAggregateAccumulator {/// Example: `CORR(x, y)` stores two arguments here, while `SUM(x)` stores one.arguments: Vec<AggregateArgument>,- /// Optional `FILTER` expression for this accumulator.+ /// Optional `FILTER` clause for this accumulator.////// Example: `SUM(x) FILTER (WHERE x > 10)` stores the `x > 10` predicate.- filter: Option<Arc<dyn PhysicalExpr>>,+ filter: Option<AggregateFilter>,/// Accumulator state for all groups for one aggregate expression.accumulator: Box<dyn GroupsAccumulator>,@@ -736,14 +736,15 @@ fn compact_group_indices(impl HashAggregateAccumulator {pub(super) fn new(aggregate_expr: Arc<AggregateFunctionExpr>,- arguments: Vec<Arc<dyn PhysicalExpr>>,+ arguments: Vec<AggregateArgument>,filter: Option<Arc<dyn PhysicalExpr>>,accumulator: Box<dyn GroupsAccumulator>,submetrics: Arc<dyn AggregateMetrics>,) -> Self {+ let filter = filter.map(|predicate| AggregateFilter::new(predicate, &arguments));Self {aggregate_expr,- arguments: arguments.into_iter().map(AggregateArgument::new).collect(),+ arguments,filter,accumulator,submetrics,@@ -759,9 +760,11 @@ impl HashAggregateAccumulator {Arc::clone(&self.aggregate_expr),self.arguments.iter()- .map(|argument| Arc::clone(argument.expr()))+ .map(AggregateArgument::empty_like).collect(),- self.filter.clone(),+ self.filter+ .as_ref()+ .map(|filter| Arc::clone(filter.predicate())),accumulator,Arc::clone(&self.submetrics),))@@ -772,6 +775,8 @@ impl HashAggregateAccumulator {/// For example, `AVG(2 / x) FILTER (WHERE x > 0)` evaluates `x > 0`/// first, then evaluates `2 / x` against a compact batch containing only/// selected rows. Filtered rows won't trigger errors such as divide by zero.+ /// Only the columns that the arguments read are copied into the compact+ /// batch (see [`AggregateFilter::select`]).////// Before updating [`GroupsAccumulator`], the retained selection is used to/// compact the matching group IDs and is not passed through.@@ -779,32 +784,28 @@ impl HashAggregateAccumulator {&mut self,batch: &RecordBatch,) -> Result<CompactedAccumulatorArgs> {- let selection = self.evaluate_filter(batch)?;- let selected_rows = selection.as_ref().map(|selection| selection.true_count());- let filtered_batch = match (selection.as_ref(), selected_rows) {- (Some(selection), Some(selected_rows))- if selected_rows > 0 && selected_rows < batch.num_rows() =>- {- Some(filter_record_batch(batch, selection)?)+ let (selection, argument_batch) = match &mut self.filter {+ None => (None, Some(Cow::Borrowed(batch))),+ Some(filter) => {+ let selection = filter.evaluate(batch)?;+ let argument_batch = match selection.true_count() {+ 0 => None,+ selected_rows if selected_rows == batch.num_rows() => {+ Some(Cow::Borrowed(batch))+ }+ _ => Some(Cow::Owned(filter.select(batch, &selection)?)),+ };+ (Some(selection), argument_batch)}- _ => None,- };- let argument_batch = match selected_rows {- None => Some(batch),- Some(0) => None,- Some(selected_rows) if selected_rows == batch.num_rows() => Some(batch),- Some(_) => filtered_batch.as_ref(),};let arguments = self.arguments.iter_mut().map(|argument| {- if let Some(argument_batch) = argument_batch {+ if let Some(argument_batch) = &argument_batch {argument.evaluate(argument_batch)} else {- let data_type =- argument.expr().data_type(batch.schema_ref().as_ref())?;- Ok(new_empty_array(&data_type))+ Ok(argument.null_array(0))}}).collect::<Result<_>>()?;@@ -824,30 +825,38 @@ impl HashAggregateAccumulator {&mut self,batch: &RecordBatch,) -> Result<RowAlignedAccumulatorArgs> {- let filter = self.evaluate_filter(batch)?;- let selection = filter.as_ref();+ let Some(filter) = &mut self.filter else {+ let arguments = self+ .arguments+ .iter_mut()+ .map(|argument| argument.evaluate(batch))+ .collect::<Result<_>>()?;+ return Ok(RowAlignedAccumulatorArgs {+ arguments,+ filter: None,+ });+ };++ let selection = filter.evaluate(batch)?;+ if !selection.has_true() {+ // No argument will be evaluated, so avoid constructing an empty+ // selected batch (or slicing every unused input column).+ return Ok(RowAlignedAccumulatorArgs {+ arguments: self.null_arguments(batch.num_rows()),+ filter: Some(selection),+ });+ }+ let selected = filter.select(batch, &selection)?;let arguments = self.arguments.iter_mut()- .map(|argument| match selection {- Some(selection) => argument.evaluate_selection(batch, selection),- None => argument.evaluate(batch),- })+ .map(|argument| argument.evaluate_selection(batch, &selection, &selected)).collect::<Result<_>>()?;- Ok(RowAlignedAccumulatorArgs { arguments, filter })- }-- fn evaluate_filter(&self, batch: &RecordBatch) -> Result<Option<BooleanArray>> {- self.filter- .as_ref()- .map(|filter| {- filter- .evaluate(batch)- .and_then(|value| value.into_array(batch.num_rows()))- .map(|filter| filter.as_boolean().clone())- })- .transpose()+ Ok(RowAlignedAccumulatorArgs {+ arguments,+ filter: Some(selection),+ })}pub(super) fn size(&self) -> usize {@@ -919,17 +928,10 @@ impl HashAggregateAccumulator {.convert_to_state(&values.arguments, values.filter.as_ref())}- pub(super) fn null_arguments(- &self,- input_schema: &SchemaRef,- num_rows: usize,- ) -> Result<Vec<ArrayRef>> {+ pub(super) fn null_arguments(&self, num_rows: usize) -> Vec<ArrayRef> {self.arguments.iter()- .map(|argument| {- let data_type = argument.expr().data_type(input_schema)?;- Ok(new_null_array(&data_type, num_rows))- })+ .map(|argument| argument.null_array(num_rows)).collect()}}@@ -1160,7 +1162,7 @@ mod tests {create_group_accumulator(&aggregate_expr, Arc::clone(&submetrics))?;Ok(HashAggregateAccumulator::new(aggregate_expr,- vec![argument],+ vec![AggregateArgument::try_new(argument, schema)?],Some(Arc::new(Column::new(filter_name, filter_index))),accumulator,submetrics,diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rsindex dbe572c615..52ac5f7c76 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs@@ -14,6 +14,7 @@ use datafusion_execution::memory_pool::proxy::VecAllocExt;use datafusion_expr::{AggregateMetrics, EmitTo};use crate::PhysicalExpr;+use crate::aggregates::aggregate_argument::AggregateArgument;use crate::aggregates::group_values::{AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,GroupByMetrics, GroupValues, new_group_values,@@ -191,6 +192,10 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {.map(|(((agg_expr, arguments), filter), submetrics)| {let accumulator =create_group_accumulator(agg_expr, Arc::clone(submetrics))?;+ let arguments = arguments+ .into_iter()+ .map(|expr| AggregateArgument::try_new(expr, input_schema))+ .collect::<Result<_>>()?;Ok(AggregateAccumulator::new(Arc::clone(agg_expr),arguments,diff --git a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs b/datafusion/physical-plan/src/aggregates/aggregate_stream.rsindex 7b43332c1d..9c29f39427 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs@@ -4,7 +4,7 @@//! Aggregate without grouping columns-use crate::aggregates::aggregate_argument::AggregateArgument;+use crate::aggregates::aggregate_argument::{AggregateArgument, AggregateFilter};use crate::aggregates::group_values::{AccumulatorPhase, AggregateAccumulatorMetrics, AggregateArgumentMetrics,aggregate_sub_metrics,@@ -33,7 +33,6 @@ use std::sync::Arc;use std::task::{Context, Poll};use super::AggregateExec;-use crate::filter::batch_filter;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};use futures::stream::{Stream, StreamExt};@@ -56,7 +55,7 @@ struct AggregateStreamInner {mode: AggregateMode,input: SendableRecordBatchStream,aggregate_arguments: Vec<Vec<AggregateArgument>>,- filter_expressions: Arc<[Option<Arc<dyn PhysicalExpr>>]>,+ filters: Vec<Option<AggregateFilter>>,aggregate_argument_metrics: AggregateArgumentMetrics,aggregate_accumulator_metrics: AggregateAccumulatorMetrics,@@ -289,14 +288,29 @@ impl AggregateStream {let baseline_metrics = BaselineMetrics::new(&agg.metrics, partition);let input = agg.input.execute(partition, Arc::clone(context))?;-- let aggregate_arguments = aggregate_expressions(agg.aggr_expr(), &agg.mode, 0)?- .into_iter()- .map(|exprs| exprs.into_iter().map(AggregateArgument::new).collect())- .collect();- let filter_expressions = match agg.mode.input_mode() {- AggregateInputMode::Raw => agg_filter_expr,- AggregateInputMode::Partial => vec![None; agg.aggr_expr().len()].into(),+ let input_schema = input.schema();++ let aggregate_arguments: Vec<Vec<_>> =+ aggregate_expressions(agg.aggr_expr(), &agg.mode, 0)?+ .into_iter()+ .map(|exprs| {+ exprs+ .into_iter()+ .map(|expr| AggregateArgument::try_new(expr, &input_schema))+ .collect::<Result<Vec<_>>>()+ })+ .collect::<Result<_>>()?;+ let filters = match agg.mode.input_mode() {+ AggregateInputMode::Raw => agg_filter_expr+ .iter()+ .zip(&aggregate_arguments)+ .map(|(filter, arguments)| {+ filter.as_ref().map(|predicate| {+ AggregateFilter::new(Arc::clone(predicate), arguments)+ })+ })+ .collect(),+ AggregateInputMode::Partial => agg.aggr_expr().iter().map(|_| None).collect(),};let aggregate_labels = agg.aggr_expr()@@ -355,7 +369,7 @@ impl AggregateStream {input,baseline_metrics,aggregate_arguments,- filter_expressions,+ filters,aggregate_argument_metrics,aggregate_accumulator_metrics,accumulators,@@ -380,7 +394,7 @@ impl AggregateStream {&batch,&mut this.accumulators,&mut this.aggregate_arguments,- &this.filter_expressions,+ &mut this.filters,&this.aggregate_argument_metrics,&this.aggregate_accumulator_metrics,)@@ -470,14 +484,15 @@ fn aggregate_batch(batch: &RecordBatch,accumulators: &mut [AccumulatorItem],arguments: &mut [Vec<AggregateArgument>],- filters: &[Option<Arc<dyn PhysicalExpr>>],+ filters: &mut [Option<AggregateFilter>],aggregate_argument_metrics: &AggregateArgumentMetrics,aggregate_accumulator_metrics: &AggregateAccumulatorMetrics,) -> Result<usize> {let mut allocated = 0usize;// 1.1 iterate accumulators and respective expressions together- // 1.2 filter the batch if necessary+ // 1.2 select the rows passing the aggregate FILTER, if any (only the+ // columns the arguments read are copied)// 1.3 evaluate expressions// 1.4 update / merge accumulators with the expressions' values@@ -491,7 +506,10 @@ fn aggregate_batch(// 1.2 and 1.3let values = aggregate_argument_metrics.time(index, || {let batch = match filter {- Some(filter) => Cow::Owned(batch_filter(batch, filter)?),+ Some(filter) => {+ let selection = filter.evaluate(batch)?;+ Cow::Owned(filter.select(batch, &selection)?)+ }None => Cow::Borrowed(batch),};argumentsdiff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 39389ad77d..0943c780e1 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -7220,6 +7220,247 @@ mod tests {Ok(())}+ /// Aggregate argument that reads `column` and records the data types of+ /// the columns of every batch it is evaluated against.+ #[derive(Debug)]+ struct RecordingArgument {+ column: Arc<dyn PhysicalExpr>,+ batch_types: Arc<Mutex<Vec<Vec<DataType>>>>,+ }++ impl PartialEq for RecordingArgument {+ fn eq(&self, other: &Self) -> bool {+ self.column.eq(&other.column)+ }+ }++ impl Eq for RecordingArgument {}++ impl std::hash::Hash for RecordingArgument {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.column.hash(state);+ }+ }++ impl std::fmt::Display for RecordingArgument {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "recording({})", self.column)+ }+ }++ impl PhysicalExpr for RecordingArgument {+ fn data_type(&self, input_schema: &Schema) -> Result<DataType> {+ self.column.data_type(input_schema)+ }++ fn nullable(&self, input_schema: &Schema) -> Result<bool> {+ self.column.nullable(input_schema)+ }++ fn evaluate(+ &self,+ batch: &RecordBatch,+ ) -> Result<datafusion_expr::ColumnarValue> {+ self.batch_types.lock().push(+ batch+ .columns()+ .iter()+ .map(|column| column.data_type().clone())+ .collect(),+ );+ self.column.evaluate(batch)+ }++ fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {+ vec![&self.column]+ }++ fn with_new_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ Ok(Arc::new(Self {+ column: children.swap_remove(0),+ batch_types: Arc::clone(&self.batch_types),+ }))+ }++ fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "recording({})", self.column)+ }+ }++ /// An aggregate `FILTER` must copy only the input columns that the+ /// aggregate's arguments read: the other columns reach the arguments as+ /// `Null` placeholders. Covers the ungrouped stream, both compacting hash+ /// aggregation paths, and skip-partial state conversion.+ #[tokio::test]+ async fn aggregate_filter_copies_only_argument_columns() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("key", DataType::Int32, true),+ Field::new("val", DataType::Int64, true),+ Field::new("payload", DataType::Utf8, true),+ Field::new("keep", DataType::Boolean, true),+ ]));+ // Every batch keeps some rows and rejects others, so each path filters.+ let batch = |keys: [i32; 4], vals: [i64; 4], keep: [Option<bool>; 4]| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(keys.to_vec())),+ Arc::new(Int64Array::from(vals.to_vec())),+ Arc::new(StringArray::from(vec!["unused"; 4])),+ Arc::new(BooleanArray::from(keep.to_vec())),+ ],+ )+ };+ let input_data = vec![+ batch(+ [1, 2, 3, 4],+ [10, 20, 30, 40],+ [Some(true), Some(false), None, Some(true)],+ )?,+ batch(+ [5, 6, 7, 8],+ [50, 60, 70, 80],+ [Some(false), Some(true), Some(false), Some(true)],+ )?,+ ];+ let argument_columns_only = vec![+ DataType::Null,+ DataType::Int64,+ DataType::Null,+ DataType::Null,+ ];++ let skip_partial_config = SessionConfig::default()+ .set(+ "datafusion.execution.skip_partial_aggregation_probe_rows_threshold",+ &ScalarValue::Int64(Some(2)),+ )+ .set(+ "datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",+ &ScalarValue::Float64(Some(0.1)),+ );+ // (mode, grouped, session config, rows converted to state by skip-partial)+ let cases = [+ (AggregateMode::Single, false, SessionConfig::default(), 0),+ (AggregateMode::Single, true, SessionConfig::default(), 0),+ (AggregateMode::Partial, true, SessionConfig::default(), 0),+ (AggregateMode::Partial, true, skip_partial_config, 4),+ ];+ // Every case evaluates the argument once per input batch, against the+ // selected rows of the argument column only.+ let expected_batch_types = cases+ .iter()+ .map(|(mode, grouped, _, skipped_rows)| {+ let batch_types = vec![argument_columns_only.clone(); input_data.len()];+ (*mode, *grouped, *skipped_rows, batch_types)+ })+ .collect::<Vec<_>>();+ let mut outputs = vec![];+ let mut batch_types_per_case = vec![];+ for (mode, grouped, session_config, skipped_rows) in cases {+ let batch_types = Arc::new(Mutex::new(vec![]));+ let argument: Arc<dyn PhysicalExpr> = Arc::new(RecordingArgument {+ column: col("val", &schema)?,+ batch_types: Arc::clone(&batch_types),+ });+ let aggr_expr = vec![+ AggregateExprBuilder::new(sum_udaf(), vec![argument])+ .schema(Arc::clone(&schema))+ .alias("SUM(val)")+ .build()+ .map(Arc::new)?,+ ];+ let group_by = if grouped {+ PhysicalGroupBy::new_single(vec![(+ col("key", &schema)?,+ "key".to_string(),+ )])+ } else {+ PhysicalGroupBy::default()+ };+ let input = TestMemoryExec::try_new_exec(+ std::slice::from_ref(&input_data),+ Arc::clone(&schema),+ None,+ )?;+ let aggregate_exec = Arc::new(AggregateExec::try_new(+ mode,+ group_by,+ aggr_expr,+ vec![Some(col("keep", &schema)?)],+ input,+ Arc::clone(&schema),+ )?);++ let ctx =+ Arc::new(TaskContext::default().with_session_config(session_config));+ let output = collect(aggregate_exec.execute(0, ctx)?).await?;+ outputs.push(batches_to_sort_string(&output));++ let metrics = aggregate_exec.metrics().unwrap();+ assert_eq!(+ metrics+ .sum_by_name("skipped_aggregation_rows")+ .map_or(0, |rows| rows.as_usize()),+ skipped_rows,+ "{mode:?}"+ );+ let batch_types = batch_types.lock().clone();+ batch_types_per_case.push((mode, grouped, skipped_rows, batch_types));+ }++ assert_eq!(batch_types_per_case, expected_batch_types);++ assert_snapshot!(outputs.join("\n"), @r"+ +----------++ | SUM(val) |+ +----------++ | 190 |+ +----------++ +-----+----------++ | key | SUM(val) |+ +-----+----------++ | 1 | 10 |+ | 2 | |+ | 3 | |+ | 4 | 40 |+ | 5 | |+ | 6 | 60 |+ | 7 | |+ | 8 | 80 |+ +-----+----------++ +-----+---------------++ | key | SUM(val)[sum] |+ +-----+---------------++ | 1 | 10 |+ | 2 | |+ | 3 | |+ | 4 | 40 |+ | 5 | |+ | 6 | 60 |+ | 7 | |+ | 8 | 80 |+ +-----+---------------++ +-----+---------------++ | key | SUM(val)[sum] |+ +-----+---------------++ | 1 | 10 |+ | 2 | |+ | 3 | |+ | 4 | 40 |+ | 5 | |+ | 6 | 60 |+ | 7 | |+ | 8 | 80 |+ +-----+---------------++ ");++ Ok(())+ }+#[tokio::test]async fn test_partial_hash_stream_skip_aggregation_after_threshold() -> Result<()> {let schema = Arc::new(Schema::new(vec![diff --git a/datafusion/sqllogictest/test_files/aggregate_filter_columns.slt b/datafusion/sqllogictest/test_files/aggregate_filter_columns.sltnew file mode 100644index 0000000000..65a983cc99--- /dev/null+++ b/datafusion/sqllogictest/test_files/aggregate_filter_columns.slt@@ -0,0 +1,295 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++# Many aggregates with FILTER over a wide input. For the rows its FILTER+# selects, each aggregate copies only the input columns its arguments read.+# Covers ungrouped, grouped and skip-partial aggregation, one and several+# partitions, multiple batches per partition, NULL and all-false FILTERs, and+# arguments that fail for rows the FILTER rejects.++statement ok+CREATE TABLE agg_filter_wide AS+SELECT+ CAST(v % 5 AS INT) AS g,+ CASE WHEN v % 7 = 0 THEN NULL ELSE CAST(v % 11 AS INT) END AS a,+ v * 3 AS b,+ v / 4.0 AS c,+ CASE WHEN v % 17 = 0 THEN NULL ELSE 'str' || CAST(v % 13 AS VARCHAR) END AS s,+ CAST(v AS DECIMAL(10, 2)) / 100 AS d,+ CASE WHEN v % 3 = 0 THEN NULL ELSE v % 2 = 0 END AS flag+FROM generate_series(1, 2000) AS t(v);++statement ok+SET datafusion.execution.batch_size = 100;++statement ok+SET datafusion.execution.target_partitions = 1;++# Single aggregation without GROUP BY. `10 / a` fails for a = 0, which the+# FILTER rejects.+query IIIIIRTRIIIIIIRT+SELECT+ count(*) FILTER (WHERE a < 3),+ count(a) FILTER (WHERE b > 3000),+ count(s) FILTER (WHERE flag),+ sum(b) FILTER (WHERE a BETWEEN 3 AND 6),+ sum(10 / a) FILTER (WHERE a <> 0),+ avg(c) FILTER (WHERE s LIKE 'str1%'),+ min(s) FILTER (WHERE a = 5),+ max(d) FILTER (WHERE flag IS NOT TRUE),+ count(DISTINCT a) FILTER (WHERE b % 2 = 0),+ sum(DISTINCT a) FILTER (WHERE c > 100),+ count(*) FILTER (WHERE a IS NULL),+ count(*) FILTER (WHERE a > 100),+ sum(b) FILTER (WHERE a > 100),+ sum(a) FILTER (WHERE flag),+ sum(c) FILTER (WHERE b > 0),+ first_value(s ORDER BY c DESC) FILTER (WHERE a = 2)+FROM agg_filter_wide;+----+468 857 628 1871142 4211 250.31908462867 str0 19.990000 11 55 285 0 NULL 2867 500250 str4++statement ok+SET datafusion.execution.target_partitions = 4;++# Partial and final aggregation without GROUP BY+query IIIIIRTRIIIIIIRT+SELECT+ count(*) FILTER (WHERE a < 3),+ count(a) FILTER (WHERE b > 3000),+ count(s) FILTER (WHERE flag),+ sum(b) FILTER (WHERE a BETWEEN 3 AND 6),+ sum(10 / a) FILTER (WHERE a <> 0),+ avg(c) FILTER (WHERE s LIKE 'str1%'),+ min(s) FILTER (WHERE a = 5),+ max(d) FILTER (WHERE flag IS NOT TRUE),+ count(DISTINCT a) FILTER (WHERE b % 2 = 0),+ sum(DISTINCT a) FILTER (WHERE c > 100),+ count(*) FILTER (WHERE a IS NULL),+ count(*) FILTER (WHERE a > 100),+ sum(b) FILTER (WHERE a > 100),+ sum(a) FILTER (WHERE flag),+ sum(c) FILTER (WHERE b > 0),+ first_value(s ORDER BY c DESC) FILTER (WHERE a = 2)+FROM agg_filter_wide;+----+468 857 628 1871142 4211 250.31908462867 str0 19.990000 11 55 285 0 NULL 2867 500250 str4++# Grouped aggregation+query IIIIIIRTRIIIIIIRT+SELECT g,+ count(*) FILTER (WHERE a < 3),+ count(a) FILTER (WHERE b > 3000),+ count(s) FILTER (WHERE flag),+ sum(b) FILTER (WHERE a BETWEEN 3 AND 6),+ sum(10 / a) FILTER (WHERE a <> 0),+ avg(c) FILTER (WHERE s LIKE 'str1%'),+ min(s) FILTER (WHERE a = 5),+ max(d) FILTER (WHERE flag IS NOT TRUE),+ count(DISTINCT a) FILTER (WHERE b % 2 = 0),+ sum(DISTINCT a) FILTER (WHERE c > 100),+ count(*) FILTER (WHERE a IS NULL),+ count(*) FILTER (WHERE a > 100),+ sum(b) FILTER (WHERE a > 100),+ sum(a) FILTER (WHERE flag),+ sum(c) FILTER (WHERE b > 0),+ first_value(s ORDER BY c DESC) FILTER (WHERE a = 2)+FROM agg_filter_wide+GROUP BY g+ORDER BY g;+----+0 92 171 126 375315 836 250.99358974359 str0 19.950000 11 55 57 0 NULL 585 100250 str7+1 94 171 126 377808 839 248.17025862069 str0 19.910000 11 55 57 0 NULL 570 99850 str8+2 95 172 125 374295 852 251.814655172414 str0 19.970000 11 55 57 0 NULL 562 99950 str6+3 94 171 125 370671 845 250.760964912281 str0 19.980000 11 55 57 0 NULL 572 100050 str4+4 93 172 126 373053 839 249.85775862069 str0 19.990000 11 55 57 0 NULL 578 100150 str12++# Ordered aggregates read their ORDER BY columns, which no argument reads+query T?I+SELECT+ string_agg(s, ',' ORDER BY d DESC) FILTER (WHERE b < 30),+ array_agg(a ORDER BY c) FILTER (WHERE b BETWEEN 30 AND 60),+ last_value(a ORDER BY b) FILTER (WHERE s = 'str3')+FROM agg_filter_wide;+----+str9,str8,str7,str6,str5,str4,str3,str2,str1 [10, 0, 1, 2, NULL, 4, 5, 6, 7, 8, 9] 1++query IT?+SELECT g,+ string_agg(s, ',' ORDER BY d DESC) FILTER (WHERE b < 60),+ array_agg(a ORDER BY c DESC) FILTER (WHERE b BETWEEN 60 AND 120)+FROM agg_filter_wide+GROUP BY g+ORDER BY g;+----+0 str2,str10,str5 [7, NULL, 8, 3, 9]+1 str3,str11,str6,str1 [3, 9, 4, NULL]+2 str12,str7,str2 [4, 10, 5, 0]+3 str5,str0,str8,str3 [5, 0, NULL, 1]+4 str6,str1,str9,str4 [6, 1, 7, 2]++# Arguments that read every input column of the aggregate+query IIR+SELECT+ sum(a + b) FILTER (WHERE a > 5),+ count(*) FILTER (WHERE b < 100),+ avg(b - a) FILTER (WHERE a <> 0 AND b % 2 = 0)+FROM (SELECT a, b FROM agg_filter_wide);+----+2351849 33 2997.5++# Skip partial aggregation: arguments are converted to state row by row+statement ok+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;++query IIIIRIRI+SELECT count(*), sum(c1), sum(c2), sum(s1), sum(s2), sum(s3), sum(s4), sum(c3)+FROM (+ SELECT b,+ count(*) FILTER (WHERE a < 3) AS c1,+ count(s) FILTER (WHERE flag) AS c2,+ sum(10 / a) FILTER (WHERE a <> 0) AS s1,+ sum(c) FILTER (WHERE s LIKE 'str1%') AS s2,+ sum(a) FILTER (WHERE a > 100) AS s3,+ sum(d) FILTER (WHERE flag IS NOT TRUE) AS s4,+ count(DISTINCT a) FILTER (WHERE g = 1) AS c3+ FROM agg_filter_wide+ GROUP BY b+);+----+2000 468 628 4211 144934.75 NULL 13336.660000 343++# Multi-argument aggregates share the selected batch during skip-partial+# conversion. Both arguments read columns distinct from the filter and key.+query IRR+SELECT g,+ corr(b, c) FILTER (WHERE flag),+ corr(b, c) FILTER (WHERE a > 100)+FROM agg_filter_wide+GROUP BY g+ORDER BY g;+----+0 1 NULL+1 1 NULL+2 1 NULL+3 1 NULL+4 1 NULL++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_rows_threshold;++statement ok+RESET datafusion.execution.batch_size;++statement ok+DROP TABLE agg_filter_wide;++# Boolean input columns keep these FILTERs observable at execution time.+# In particular, keep_nullable has no false values, but its NULLs must reject+# rows before division. Repeated batches exercise cached argument types.+statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+SET datafusion.execution.batch_size = 4;++statement ok+CREATE TABLE agg_filter_masks AS+SELECT+ value AS g,+ CASE WHEN value % 2 = 0 THEN 0 ELSE 5 END AS v,+ value > 0 AS keep_all,+ value < 0 AS keep_none,+ CASE WHEN value > 0 THEN NULL ELSE false END AS keep_null,+ CASE WHEN value % 2 = 0 THEN NULL ELSE true END AS keep_nullable+FROM generate_series(1, 32);++query IIRIIIIIII+SELECT+ count(*) FILTER (WHERE keep_all),+ sum(v) FILTER (WHERE keep_all),+ avg(CAST(v AS DECIMAL(10, 2))) FILTER (WHERE keep_all),+ count(*) FILTER (WHERE keep_none),+ sum(10 / v) FILTER (WHERE keep_none),+ sum(10 / v) FILTER (WHERE keep_null),+ count(*) FILTER (WHERE keep_nullable),+ sum(10 / v) FILTER (WHERE keep_nullable),+ sum(7) FILTER (WHERE keep_nullable),+ count(v) FILTER (WHERE keep_nullable)+FROM agg_filter_masks;+----+32 80 2.500000 0 NULL NULL 16 32 112 16++# Ordinary grouped aggregation handles empty and complete selections.+statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 100000;++query IIIRIIIIIII+SELECT count(*), sum(c1), sum(s1), avg(a1), sum(c0), sum(s0), sum(sn), sum(cn), sum(ss), sum(sl), sum(cv)+FROM (+ SELECT g,+ count(*) FILTER (WHERE keep_all) AS c1,+ sum(v) FILTER (WHERE keep_all) AS s1,+ avg(CAST(v AS DECIMAL(10, 2))) FILTER (WHERE keep_all) AS a1,+ count(*) FILTER (WHERE keep_none) AS c0,+ sum(10 / v) FILTER (WHERE keep_none) AS s0,+ sum(10 / v) FILTER (WHERE keep_null) AS sn,+ count(*) FILTER (WHERE keep_nullable) AS cn,+ sum(10 / v) FILTER (WHERE keep_nullable) AS ss,+ sum(7) FILTER (WHERE keep_nullable) AS sl,+ count(v) FILTER (WHERE keep_nullable) AS cv+ FROM agg_filter_masks+ GROUP BY g+);+----+32 32 80 2.5000000000 0 NULL NULL 16 32 112 16++# Skip-partial must preserve row positions for the same selections. Decimal+# AVG also exercises an argument type different from the aggregate output.+statement ok+SET datafusion.execution.skip_partial_aggregation_probe_rows_threshold = 0;++statement ok+SET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold = 0.0;++query IIIRIIIIIII+SELECT count(*), sum(c1), sum(s1), avg(a1), sum(c0), sum(s0), sum(sn), sum(cn), sum(ss), sum(sl), sum(cv)+FROM (+ SELECT g,+ count(*) FILTER (WHERE keep_all) AS c1,+ sum(v) FILTER (WHERE keep_all) AS s1,+ avg(CAST(v AS DECIMAL(10, 2))) FILTER (WHERE keep_all) AS a1,+ count(*) FILTER (WHERE keep_none) AS c0,+ sum(10 / v) FILTER (WHERE keep_none) AS s0,+ sum(10 / v) FILTER (WHERE keep_null) AS sn,+ count(*) FILTER (WHERE keep_nullable) AS cn,+ sum(10 / v) FILTER (WHERE keep_nullable) AS ss,+ sum(7) FILTER (WHERE keep_nullable) AS sl,+ count(v) FILTER (WHERE keep_nullable) AS cv+ FROM agg_filter_masks+ GROUP BY g+);+----+32 32 80 2.5000000000 0 NULL NULL 16 32 112 16++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_ratio_threshold;++statement ok+RESET datafusion.execution.skip_partial_aggregation_probe_rows_threshold;++statement ok+RESET datafusion.execution.batch_size;++statement ok+DROP TABLE agg_filter_masks;