Skip to content

Apache DataFusion SQL Query Engine

keep fetch when merge FilterExec in FilterPushdown (#21070)

Huaijin8f721a6500ff
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index 8f430f7753..9f3dffd230 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -532,6 +532,240 @@ fn test_filter_with_projection() {
);
}
+#[test]
+fn test_filter_collapse_outer_fetch_preserved() {
+ // When the outer filter has fetch and inner does not, the merged filter should preserve fetch
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let predicate1 = col_lit_predicate("a", "foo", &schema());
+ let filter1 = Arc::new(FilterExec::try_new(predicate1, scan).unwrap());
+ let predicate2 = col_lit_predicate("b", "bar", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate2, filter1)
+ .with_fetch(Some(10))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: b@1 = bar, fetch=10
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - FilterExec: b@1 = bar AND a@0 = foo, fetch=10
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+#[test]
+fn test_filter_collapse_inner_fetch_preserved() {
+ // When the inner filter has fetch and outer does not, the merged filter should preserve fetch
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let predicate1 = col_lit_predicate("a", "foo", &schema());
+ let filter1 = Arc::new(
+ FilterExecBuilder::new(predicate1, scan)
+ .with_fetch(Some(5))
+ .build()
+ .unwrap(),
+ );
+ let predicate2 = col_lit_predicate("b", "bar", &schema());
+ let plan = Arc::new(FilterExec::try_new(predicate2, filter1).unwrap());
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: b@1 = bar
+ - FilterExec: a@0 = foo, fetch=5
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - FilterExec: b@1 = bar AND a@0 = foo, fetch=5
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+#[test]
+fn test_filter_collapse_both_fetch_uses_minimum() {
+ // When both filters have fetch, the merged filter should use the smaller (tighter) fetch.
+ // Inner fetch=5 is tighter than outer fetch=10, so the result should be fetch=5.
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let predicate1 = col_lit_predicate("a", "foo", &schema());
+ let filter1 = Arc::new(
+ FilterExecBuilder::new(predicate1, scan)
+ .with_fetch(Some(5))
+ .build()
+ .unwrap(),
+ );
+ let predicate2 = col_lit_predicate("b", "bar", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate2, filter1)
+ .with_fetch(Some(10))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: b@1 = bar, fetch=10
+ - FilterExec: a@0 = foo, fetch=5
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - FilterExec: b@1 = bar AND a@0 = foo, fetch=5
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_fully_pushed_to_scan() {
+ // When a FilterExec has a fetch limit and all predicates are pushed down
+ // to a supportive DataSourceExec, the FilterExec is removed and the fetch
+ // must be propagated to the DataSourceExec.
+ let scan = TestScanBuilder::new(schema()).with_support(true).build();
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, scan)
+ .with_fetch(Some(10))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, fetch=10
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], limit=10, file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_and_projection_fully_pushed_to_scan() {
+ // When a FilterExec has both fetch and projection, and all predicates are
+ // pushed down, the filter is replaced by a ProjectionExec and the fetch
+ // must still be propagated to the DataSourceExec.
+ let scan = TestScanBuilder::new(schema()).with_support(true).build();
+ let projection = vec![1, 0];
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, scan)
+ .with_fetch(Some(5))
+ .apply_projection(Some(projection))
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, projection=[b@1, a@0], fetch=5
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], limit=5, file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_partially_pushed_to_scan() {
+ // When a FilterExec has fetch and only some predicates are pushed down,
+ // the FilterExec remains with the unpushed predicate and keeps its fetch.
+ let scan = TestScanBuilder::new(schema()).with_support(true).build();
+ let pushed_predicate = col_lit_predicate("a", "foo", &schema());
+ let volatile_predicate = {
+ let cfg = Arc::new(ConfigOptions::default());
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new_with_schema("a", &schema()).unwrap()),
+ Operator::Eq,
+ Arc::new(
+ ScalarFunctionExpr::try_new(
+ Arc::new(ScalarUDF::from(RandomFunc::new())),
+ vec![],
+ &schema(),
+ cfg,
+ )
+ .unwrap(),
+ ),
+ )) as Arc<dyn PhysicalExpr>
+ };
+ // Combine: a = 'foo' AND a = random()
+ let combined = Arc::new(BinaryExpr::new(
+ pushed_predicate,
+ Operator::And,
+ volatile_predicate,
+ )) as Arc<dyn PhysicalExpr>;
+ let plan = Arc::new(
+ FilterExecBuilder::new(combined, scan)
+ .with_fetch(Some(7))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo AND a@0 = random(), fetch=7
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - FilterExec: a@0 = random(), fetch=7
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_not_pushed_to_unsupportive_scan() {
+ // When the DataSourceExec does not support pushdown, the FilterExec
+ // remains unchanged with its fetch intact.
+ let scan = TestScanBuilder::new(schema()).with_support(false).build();
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, scan)
+ .with_fetch(Some(3))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @r"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, fetch=3
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - FilterExec: a@0 = foo, fetch=3
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
#[test]
fn test_push_down_through_transparent_nodes() {
// expect the predicate to be pushed down into the DataSource
@@ -2087,6 +2321,116 @@ fn test_filter_pushdown_through_union_does_not_support() {
);
}
+#[test]
+fn test_filter_with_fetch_fully_pushed_through_union() {
+ // When a FilterExec with fetch wraps a UnionExec and all predicates are
+ // pushed down, UnionExec does not support with_fetch, so a LocalLimitExec
+ // should be inserted to preserve the fetch limit.
+ let scan1 = TestScanBuilder::new(schema()).with_support(true).build();
+ let scan2 = TestScanBuilder::new(schema()).with_support(true).build();
+ let union = UnionExec::try_new(vec![scan1, scan2]).unwrap();
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, union)
+ .with_fetch(Some(10))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, fetch=10
+ - UnionExec
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - LocalLimitExec: fetch=10
+ - UnionExec
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_and_projection_fully_pushed_through_union() {
+ // When a FilterExec with both fetch and projection wraps a UnionExec and
+ // all predicates are pushed down, we should get a ProjectionExec on top of
+ // a LocalLimitExec wrapping the UnionExec.
+ let scan1 = TestScanBuilder::new(schema()).with_support(true).build();
+ let scan2 = TestScanBuilder::new(schema()).with_support(true).build();
+ let union = UnionExec::try_new(vec![scan1, scan2]).unwrap();
+ let projection = vec![1, 0];
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, union)
+ .with_fetch(Some(5))
+ .apply_projection(Some(projection))
+ .unwrap()
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, projection=[b@1, a@0], fetch=5
+ - UnionExec
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true
+ output:
+ Ok:
+ - ProjectionExec: expr=[b@1 as b, a@0 as a]
+ - LocalLimitExec: fetch=5
+ - UnionExec
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo
+ "
+ );
+}
+
+#[test]
+fn test_filter_with_fetch_not_fully_pushed_through_union() {
+ // When a FilterExec with fetch wraps a UnionExec but children don't support
+ // pushdown, the FilterExec remains with its fetch — no LocalLimitExec needed.
+ let scan1 = TestScanBuilder::new(schema()).with_support(false).build();
+ let scan2 = TestScanBuilder::new(schema()).with_support(false).build();
+ let union = UnionExec::try_new(vec![scan1, scan2]).unwrap();
+ let predicate = col_lit_predicate("a", "foo", &schema());
+ let plan = Arc::new(
+ FilterExecBuilder::new(predicate, union)
+ .with_fetch(Some(8))
+ .build()
+ .unwrap(),
+ );
+
+ insta::assert_snapshot!(
+ OptimizationTest::new(plan, FilterPushdown::new(), true),
+ @"
+ OptimizationTest:
+ input:
+ - FilterExec: a@0 = foo, fetch=8
+ - UnionExec
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ output:
+ Ok:
+ - LocalLimitExec: fetch=8
+ - UnionExec
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ - FilterExec: a@0 = foo
+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=false
+ "
+ );
+}
+
/// Schema:
/// a: String
/// b: String
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 141d9c3846..c404f8e61b 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -35,6 +35,7 @@ use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
FilterPushdownPropagation, PushedDown,
};
+use crate::limit::LocalLimitExec;
use crate::metrics::{MetricBuilder, MetricType};
use crate::projection::{
EmbeddedProjection, ProjectionExec, ProjectionExpr, make_with_child,
@@ -695,7 +696,22 @@ impl ExecutionPlan for FilterExec {
let filter_input = Arc::clone(self.input());
let new_predicate = conjunction(unhandled_filters);
let updated_node = if new_predicate.eq(&lit(true)) {
- // FilterExec is no longer needed, but we may need to leave a projection in place
+ // FilterExec is no longer needed, but we may need to leave a projection in place.
+ // If this FilterExec had a fetch limit, propagate it to the child.
+ // When the child also has a fetch, use the minimum of both to preserve
+ // the tighter constraint.
+ let filter_input = if let Some(outer_fetch) = self.fetch {
+ let effective_fetch = match filter_input.fetch() {
+ Some(inner_fetch) => outer_fetch.min(inner_fetch),
+ None => outer_fetch,
+ };
+ match filter_input.with_fetch(Some(effective_fetch)) {
+ Some(node) => node,
+ None => Arc::new(LocalLimitExec::new(filter_input, effective_fetch)),
+ }
+ } else {
+ filter_input
+ };
match self.projection().as_ref() {
Some(projection_indices) => {
let filter_child_schema = filter_input.schema();