Apache DataFusion SQL Query Engine
Allow filter pushdown through AggregateExec (#18404)
Lía Adriana076b091a4706
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rsindex b91c173226..de61149508 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs@@ -543,11 +543,11 @@ fn test_push_down_through_transparent_nodes() {}#[test]-fn test_no_pushdown_through_aggregates() {- // There are 2 important points here:- // 1. The outer filter **is not** pushed down at all because we haven't implemented pushdown support- // yet for AggregateExec.- // 2. The inner filter **is** pushed down into the DataSource.+fn test_pushdown_through_aggregates_on_grouping_columns() {+ // Test that filters on grouping columns can be pushed through AggregateExec.+ // This test has two filters:+ // 1. An inner filter (a@0 = foo) below the aggregate - gets pushed to DataSource+ // 2. An outer filter (b@1 = bar) above the aggregate - also gets pushed through because 'b' is a grouping columnlet scan = TestScanBuilder::new(schema()).with_support(true).build();let coalesce = Arc::new(CoalesceBatchesExec::new(scan, 10));@@ -586,7 +586,7 @@ fn test_no_pushdown_through_aggregates() {let predicate = col_lit_predicate("b", "bar", &schema());let plan = Arc::new(FilterExec::try_new(predicate, coalesce).unwrap());- // expect the predicate to be pushed down into the DataSource+ // Both filters should be pushed down to the DataSource since both reference grouping columnsinsta::assert_snapshot!(OptimizationTest::new(plan, FilterPushdown::new(), true),@r"@@ -600,11 +600,10 @@ fn test_no_pushdown_through_aggregates() {- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - FilterExec: b@1 = bar- - CoalesceBatchesExec: target_batch_size=100- - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([0])- - CoalesceBatchesExec: target_batch_size=10- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo+ - CoalesceBatchesExec: target_batch_size=100+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=Sorted+ - CoalesceBatchesExec: target_batch_size=10+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo AND b@1 = bar");}@@ -1892,3 +1891,445 @@ fn col_lit_predicate(Arc::new(Literal::new(scalar_value)),))}++#[tokio::test]+async fn test_aggregate_filter_pushdown() {+ // Test that filters can pass through AggregateExec even with aggregate functions+ // when the filter references grouping columns+ // Simulates: SELECT a, COUNT(b) FROM table WHERE a = 'x' GROUP BY a++ let batches =+ vec![+ record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),+ ];++ let scan = TestScanBuilder::new(schema())+ .with_support(true)+ .with_batches(batches)+ .build();++ // Create an aggregate: GROUP BY a with COUNT(b)+ let group_by = PhysicalGroupBy::new_single(vec![(+ col("a", &schema()).unwrap(),+ "a".to_string(),+ )]);++ // Add COUNT aggregate+ let count_expr =+ AggregateExprBuilder::new(count_udaf(), vec![col("b", &schema()).unwrap()])+ .schema(schema())+ .alias("count")+ .build()+ .unwrap();++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Partial,+ group_by,+ vec![count_expr.into()], // Has aggregate function+ vec![None], // No filter on the aggregate function+ Arc::clone(&scan),+ schema(),+ )+ .unwrap(),+ );++ // Add a filter on the grouping column 'a'+ let predicate = col_lit_predicate("a", "x", &schema());+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap())+ as Arc<dyn ExecutionPlan>;++ // Even with aggregate functions, filter on grouping column should be pushed through+ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: a@0 = x+ - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count], ordering_mode=Sorted+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = x+ "+ );+}++#[tokio::test]+async fn test_no_pushdown_filter_on_aggregate_result() {+ // Test that filters on aggregate results (not grouping columns) are NOT pushed through+ // SELECT a, COUNT(b) as cnt FROM table GROUP BY a HAVING cnt > 5+ // The filter on 'cnt' cannot be pushed down because it's an aggregate result++ let batches =+ vec![+ record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),+ ];++ let scan = TestScanBuilder::new(schema())+ .with_support(true)+ .with_batches(batches)+ .build();++ // Create an aggregate: GROUP BY a with COUNT(b)+ let group_by = PhysicalGroupBy::new_single(vec![(+ col("a", &schema()).unwrap(),+ "a".to_string(),+ )]);++ // Add COUNT aggregate+ let count_expr =+ AggregateExprBuilder::new(count_udaf(), vec![col("b", &schema()).unwrap()])+ .schema(schema())+ .alias("count")+ .build()+ .unwrap();++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Partial,+ group_by,+ vec![count_expr.into()],+ vec![None],+ Arc::clone(&scan),+ schema(),+ )+ .unwrap(),+ );++ // Add a filter on the aggregate output column+ // This simulates filtering on COUNT result, which should NOT be pushed through+ let agg_schema = aggregate.schema();+ let predicate = Arc::new(BinaryExpr::new(+ Arc::new(Column::new_with_schema("count[count]", &agg_schema).unwrap()),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int64(Some(5)))),+ ));+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap())+ as Arc<dyn ExecutionPlan>;++ // The filter should NOT be pushed through the aggregate since it's on an aggregate result+ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: count[count]@1 > 5+ - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - FilterExec: count[count]@1 > 5+ - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ "+ );+}++#[test]+fn test_pushdown_filter_on_non_first_grouping_column() {+ // Test that filters on non-first grouping columns are still pushed down+ // SELECT a, b, count(*) as cnt FROM table GROUP BY a, b HAVING b = 'bar'+ // The filter is on 'b' (second grouping column), should push down+ let scan = TestScanBuilder::new(schema()).with_support(true).build();++ let aggregate_expr =+ vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];++ let group_by = PhysicalGroupBy::new_single(vec![+ (col("a", &schema()).unwrap(), "a".to_string()),+ (col("b", &schema()).unwrap(), "b".to_string()),+ ]);++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ aggregate_expr.clone(),+ vec![None],+ scan,+ schema(),+ )+ .unwrap(),+ );++ let predicate = col_lit_predicate("b", "bar", &schema());+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());++ insta::assert_snapshot!(+ OptimizationTest::new(plan, FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: b@1 = bar+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar+ "+ );+}++#[test]+fn test_no_pushdown_grouping_sets_filter_on_missing_column() {+ // Test that filters on columns missing from some grouping sets are NOT pushed through+ let scan = TestScanBuilder::new(schema()).with_support(true).build();++ let aggregate_expr =+ vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];++ // Create GROUPING SETS with (a, b) and (b)+ let group_by = PhysicalGroupBy::new(+ vec![+ (col("a", &schema()).unwrap(), "a".to_string()),+ (col("b", &schema()).unwrap(), "b".to_string()),+ ],+ vec![+ (+ Arc::new(Literal::new(ScalarValue::Utf8(None))),+ "a".to_string(),+ ),+ (+ Arc::new(Literal::new(ScalarValue::Utf8(None))),+ "b".to_string(),+ ),+ ],+ vec![+ vec![false, false], // (a, b) - both present+ vec![true, false], // (b) - a is NULL, b present+ ],+ );++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ aggregate_expr.clone(),+ vec![None],+ scan,+ schema(),+ )+ .unwrap(),+ );++ // Filter on column 'a' which is missing in the second grouping set, should not be pushed down+ let predicate = col_lit_predicate("a", "foo", &schema());+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());++ insta::assert_snapshot!(+ OptimizationTest::new(plan, FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: a@0 = foo+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - FilterExec: a@0 = foo+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ "+ );+}++#[test]+fn test_pushdown_grouping_sets_filter_on_common_column() {+ // Test that filters on columns present in ALL grouping sets ARE pushed through+ let scan = TestScanBuilder::new(schema()).with_support(true).build();++ let aggregate_expr =+ vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];++ // Create GROUPING SETS with (a, b) and (b)+ let group_by = PhysicalGroupBy::new(+ vec![+ (col("a", &schema()).unwrap(), "a".to_string()),+ (col("b", &schema()).unwrap(), "b".to_string()),+ ],+ vec![+ (+ Arc::new(Literal::new(ScalarValue::Utf8(None))),+ "a".to_string(),+ ),+ (+ Arc::new(Literal::new(ScalarValue::Utf8(None))),+ "b".to_string(),+ ),+ ],+ vec![+ vec![false, false], // (a, b) - both present+ vec![true, false], // (b) - a is NULL, b present+ ],+ );++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ aggregate_expr.clone(),+ vec![None],+ scan,+ schema(),+ )+ .unwrap(),+ );++ // Filter on column 'b' which is present in all grouping sets will be pushed down+ let predicate = col_lit_predicate("b", "bar", &schema());+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());++ insta::assert_snapshot!(+ OptimizationTest::new(plan, FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: b@1 = bar+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - AggregateExec: mode=Final, gby=[(a@0 as a, b@1 as b), (NULL as a, b@1 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar+ "+ );+}++#[test]+fn test_pushdown_with_empty_group_by() {+ // Test that filters can be pushed down when GROUP BY is empty (no grouping columns)+ // SELECT count(*) as cnt FROM table WHERE a = 'foo'+ // There are no grouping columns, so the filter should still push down+ let scan = TestScanBuilder::new(schema()).with_support(true).build();++ let aggregate_expr =+ vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];++ // Empty GROUP BY - no grouping columns+ let group_by = PhysicalGroupBy::new_single(vec![]);++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ aggregate_expr.clone(),+ vec![None],+ scan,+ schema(),+ )+ .unwrap(),+ );++ // Filter on 'a'+ let predicate = col_lit_predicate("a", "foo", &schema());+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());++ // The filter should be pushed down even with empty GROUP BY+ insta::assert_snapshot!(+ OptimizationTest::new(plan, FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: a@0 = foo+ - AggregateExec: mode=Final, gby=[], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - AggregateExec: mode=Final, gby=[], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo+ "+ );+}++#[test]+fn test_pushdown_with_computed_grouping_key() {+ // Test filter pushdown with computed grouping expression+ // SELECT (c + 1.0) as c_plus_1, count(*) FROM table WHERE c > 5.0 GROUP BY (c + 1.0)++ let scan = TestScanBuilder::new(schema()).with_support(true).build();++ let predicate = Arc::new(BinaryExpr::new(+ col("c", &schema()).unwrap(),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Float64(Some(5.0)))),+ )) as Arc<dyn PhysicalExpr>;+ let filter = Arc::new(FilterExec::try_new(predicate, scan).unwrap());++ let aggregate_expr =+ vec![+ AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()])+ .schema(schema())+ .alias("cnt")+ .build()+ .map(Arc::new)+ .unwrap(),+ ];++ let c_plus_one = Arc::new(BinaryExpr::new(+ col("c", &schema()).unwrap(),+ Operator::Plus,+ Arc::new(Literal::new(ScalarValue::Float64(Some(1.0)))),+ )) as Arc<dyn PhysicalExpr>;++ let group_by =+ PhysicalGroupBy::new_single(vec![(c_plus_one, "c_plus_1".to_string())]);++ let plan = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ aggregate_expr.clone(),+ vec![None],+ filter,+ schema(),+ )+ .unwrap(),+ );++ // The filter should be pushed down because 'c' is extracted from the grouping expression (c + 1.0)+ insta::assert_snapshot!(+ OptimizationTest::new(plan, FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - AggregateExec: mode=Final, gby=[c@2 + 1 as c_plus_1], aggr=[cnt]+ - FilterExec: c@2 > 5+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - AggregateExec: mode=Final, gby=[c@2 + 1 as c_plus_1], aggr=[cnt]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=c@2 > 5+ "+ );+}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 878bccc1d1..30d1441f57 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -26,12 +26,18 @@ use crate::aggregates::{topk_stream::GroupedTopKAggregateStream,};use crate::execution_plan::{CardinalityEffect, EmissionType};+use crate::filter_pushdown::{+ ChildFilterDescription, FilterDescription, FilterPushdownPhase, PushedDownPredicate,+};use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};use crate::windows::get_ordered_partition_by_indices;use crate::{DisplayFormatType, Distribution, ExecutionPlan, InputOrderMode,SendableRecordBatchStream, Statistics,};+use datafusion_common::config::ConfigOptions;+use datafusion_physical_expr::utils::collect_columns;+use std::collections::HashSet;use arrow::array::{ArrayRef, UInt16Array, UInt32Array, UInt64Array, UInt8Array};use arrow::datatypes::{Field, Schema, SchemaRef};@@ -1025,6 +1031,88 @@ impl ExecutionPlan for AggregateExec {fn cardinality_effect(&self) -> CardinalityEffect {CardinalityEffect::LowerEqual}++ /// Push down parent filters when possible (see implementation comment for details),+ /// but do not introduce any new self filters.+ fn gather_filters_for_pushdown(+ &self,+ _phase: FilterPushdownPhase,+ parent_filters: Vec<Arc<dyn PhysicalExpr>>,+ _config: &ConfigOptions,+ ) -> Result<FilterDescription> {+ // It's safe to push down filters through aggregates when filters only reference+ // grouping columns, because such filters determine which groups to compute, not+ // *how* to compute them. Each group's aggregate values (SUM, COUNT, etc.) are+ // calculated from the same input rows regardless of whether we filter before or+ // after grouping - filtering before just eliminates entire groups early.+ // This optimization is NOT safe for filters on aggregated columns (like filtering on+ // the result of SUM or COUNT), as those require computing all groups first.++ let grouping_columns: HashSet<_> = self+ .group_by+ .expr()+ .iter()+ .flat_map(|(expr, _)| collect_columns(expr))+ .collect();++ // Analyze each filter separately to determine if it can be pushed down+ let mut safe_filters = Vec::new();+ let mut unsafe_filters = Vec::new();++ for filter in parent_filters {+ let filter_columns: HashSet<_> =+ collect_columns(&filter).into_iter().collect();++ // Check if this filter references non-grouping columns+ let references_non_grouping = !grouping_columns.is_empty()+ && !filter_columns.is_subset(&grouping_columns);++ if references_non_grouping {+ unsafe_filters.push(filter);+ continue;+ }++ // For GROUPING SETS, verify this filter's columns appear in all grouping sets+ if self.group_by.groups().len() > 1 {+ let filter_column_indices: Vec<usize> = filter_columns+ .iter()+ .filter_map(|filter_col| {+ self.group_by.expr().iter().position(|(expr, _)| {+ collect_columns(expr).contains(filter_col)+ })+ })+ .collect();++ // Check if any of this filter's columns are missing from any grouping set+ let has_missing_column = self.group_by.groups().iter().any(|null_mask| {+ filter_column_indices+ .iter()+ .any(|&idx| null_mask.get(idx) == Some(&true))+ });++ if has_missing_column {+ unsafe_filters.push(filter);+ continue;+ }+ }++ // This filter is safe to push down+ safe_filters.push(filter);+ }++ // Build child filter description with both safe and unsafe filters+ let child = self.children()[0];+ let mut child_desc = ChildFilterDescription::from_child(&safe_filters, child)?;++ // Add unsafe filters as unsupported+ child_desc.parent_filters.extend(+ unsafe_filters+ .into_iter()+ .map(PushedDownPredicate::unsupported),+ );++ Ok(FilterDescription::new().with_child(child_desc))+ }}fn create_schema(