Apache DataFusion SQL Query Engine
feat: transfer parent and dynamic filters across HashJoinExec equi-join keys (inner and semi joins) (#25255)
Jay Zhancf5d573e14f0
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex f10725fd30..783c216f20 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -24,7 +24,7 @@ use arrow::{};use arrow_schema::SortOptions;use datafusion::{- assert_batches_eq,+ assert_batches_eq, assert_batches_sorted_eq,logical_expr::Operator,physical_plan::{PhysicalExpr,@@ -51,7 +51,7 @@ use datafusion_functions_aggregate::{};use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr,- expressions::{DynamicFilterPhysicalExpr, col},+ expressions::{DynamicFilterPhysicalExpr, IsNullExpr, cast, col},utils::conjunction,};use datafusion_physical_expr::{@@ -287,7 +287,7 @@ async fn test_static_filter_pushdown_through_hash_join() {- FilterExec: a@0 = d@3- HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, d@0)]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = aa- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true, predicate=e@1 = ba+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true, predicate=d@0 = aa AND e@1 = ba");@@ -1596,7 +1596,7 @@ fn test_hashjoin_parent_filter_pushdown_same_column_names() {Ok:- HashJoinExec: mode=Partitioned, join_type=Inner, on=[(id@0, id@0)]- DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, build_val], file_type=test, pushdown_supported=true, predicate=id@0 = aa- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, probe_val], file_type=test, pushdown_supported=true, predicate=probe_val@1 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, probe_val], file_type=test, pushdown_supported=true, predicate=id@0 = aa AND probe_val@1 = x");}@@ -1752,8 +1752,10 @@ fn test_aggregate_pushdown_preserves_duplicate_grouping_columns() {assert_eq!(filters[0][1].predicate.to_string(), "id@0 = x");}-/// A semi join key that is not a plain column cannot be mapped to the other-/// side, so a filter on that output key only reaches the emitted side.+/// A semi join whose emitted-side key is an expression has no output column+/// that is a join key, so a filter on the emitted column stays on that side.+/// When the emitted-side key is a plain column, the filter is transferred to+/// the other side, rewritten over that side's key expression.#[test]fn test_hashjoin_parent_filter_pushdown_semi_join_expression_key() {use datafusion_physical_expr::expressions::CastExpr;@@ -1763,7 +1765,10 @@ fn test_hashjoin_parent_filter_pushdown_semi_join_expression_key() {let key = || Arc::new(Column::new("id", 0)) as Arc<dyn PhysicalExpr>;let cast_key =|| Arc::new(CastExpr::new(key(), DataType::Utf8, None)) as Arc<dyn PhysicalExpr>;- for on in [(cast_key(), key()), (key(), cast_key())] {+ for (on, transferred) in [+ ((cast_key(), key()), None),+ ((key(), cast_key()), Some("CAST(id@0 AS Utf8) = x")),+ ] {let join = HashJoinExec::try_new(TestScanBuilder::new(Arc::clone(&schema)).build(),TestScanBuilder::new(Arc::clone(&schema)).build(),@@ -1785,8 +1790,14 @@ fn test_hashjoin_parent_filter_pushdown_semi_join_expression_key() {.unwrap().parent_filters();assert!(matches!(filters[0][0].discriminant, PushedDown::Yes));- assert!(matches!(filters[1][0].discriminant, PushedDown::No));assert_eq!(filters[0][0].predicate.to_string(), "id@0 = x");+ match transferred {+ Some(expected) => {+ assert!(matches!(filters[1][0].discriminant, PushedDown::Yes));+ assert_eq!(filters[1][0].predicate.to_string(), expected);+ }+ None => assert!(matches!(filters[1][0].discriminant, PushedDown::No)),+ }}}@@ -1895,7 +1906,9 @@ fn test_from_child_with_allowed_indices_rejects_unresolvable_name() {}/// A join's output projection must map to child positions even when a child-/// contains multiple columns with the same name.+/// contains multiple columns with the same name. Position 0 on each side is+/// the join key, so a filter on it is also transferred to the other side's+/// key; the same-named non-key column at position 1 stays on its own side.#[test]fn test_hashjoin_parent_filter_pushdown_duplicate_child_columns() {use datafusion_physical_plan::filter_pushdown::{FilterPushdownPhase, PushedDown};@@ -1937,11 +1950,16 @@ fn test_hashjoin_parent_filter_pushdown_duplicate_child_columns() {.parent_filters();let side = input_index / 2;assert!(matches!(filters[side][0].discriminant, PushedDown::Yes));- assert!(matches!(filters[1 - side][0].discriminant, PushedDown::No));assert_eq!(filters[side][0].predicate.to_string(),format!("id@{} = x", input_index % 2));+ if input_index % 2 == 0 {+ assert!(matches!(filters[1 - side][0].discriminant, PushedDown::Yes));+ assert_eq!(filters[1 - side][0].predicate.to_string(), "id@0 = x");+ } else {+ assert!(matches!(filters[1 - side][0].discriminant, PushedDown::No));+ }}}}@@ -2151,6 +2169,698 @@ fn test_hashjoin_parent_filter_pushdown_semi_anti_join() {assert_parent_filter_remains(plan);}+/// Under `NullEqualsNull` an inner join also emits rows whose keys are NULL+/// on both sides, so `id` and `pid` are still identical in every output row+/// (both NULL or equal). A parent filter over one side's key, including an+/// `IS NULL` check, therefore transfers to the other side exactly as under+/// `NullEqualsNothing`, and the NULL-keyed matches survive the transfer.+#[tokio::test]+async fn test_hashjoin_parent_filter_transfer_null_equals_null_inner_join() {+ let build_side_schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Utf8, true),+ Field::new("build_val", DataType::Utf8, false),+ ]));+ let build_batches = vec![+ record_batch!(+ ("id", Utf8, [Some("aa"), None, Some("bb")]),+ ("build_val", Utf8, ["b1", "b2", "b3"])+ )+ .unwrap(),+ ];+ let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))+ .with_support(true)+ .with_batches(build_batches)+ .build();++ let probe_side_schema = Arc::new(Schema::new(vec![+ Field::new("pid", DataType::Utf8, true),+ Field::new("probe_val", DataType::Utf8, false),+ ]));+ let probe_batches = vec![+ record_batch!(+ ("pid", Utf8, [Some("aa"), None, Some("bb"), None]),+ ("probe_val", Utf8, ["p1", "p2", "p3", "p4"])+ )+ .unwrap(),+ ];+ let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))+ .with_support(true)+ .with_batches(probe_batches)+ .build();++ let on = vec![(+ col("id", &build_side_schema).unwrap(),+ col("pid", &probe_side_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ build_scan,+ probe_scan,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ datafusion_common::NullEquality::NullEqualsNull,+ false,+ )+ .unwrap(),+ );+ let join_schema = join.schema();++ // id = 'aa' OR id IS NULL: keeps the 'aa' match and the NULL/NULL matches+ let predicate = Arc::new(BinaryExpr::new(+ col_lit_predicate("id", "aa", &join_schema),+ Operator::Or,+ Arc::new(IsNullExpr::new(col("id", &join_schema).unwrap())),+ ));+ let plan =+ Arc::new(FilterExec::try_new(predicate, join).unwrap()) as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: id@0 = aa OR id@0 IS NULL+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, pid@0)], NullsEqual: true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, build_val], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[pid, probe_val], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, pid@0)], NullsEqual: true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, build_val], file_type=test, pushdown_supported=true, predicate=id@0 = aa OR id@0 IS NULL+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[pid, probe_val], file_type=test, pushdown_supported=true, predicate=pid@0 = aa OR pid@0 IS NULL+ "+ );++ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ let optimized = FilterPushdown::new().optimize(plan, &config).unwrap();+ let session_ctx = SessionContext::new();+ session_ctx.register_object_store(+ ObjectStoreUrl::parse("test://").unwrap().as_ref(),+ Arc::new(InMemory::new()),+ );+ let batches = collect(optimized, session_ctx.task_ctx()).await.unwrap();+ // The NULL build key matches both NULL probe keys; 'bb' is filtered out+ // on both sides and the parent filter is gone.+ #[rustfmt::skip]+ let expected = [+ "+----+-----------+-----+-----------+",+ "| id | build_val | pid | probe_val |",+ "+----+-----------+-----+-----------+",+ "| | b2 | | p2 |",+ "| | b2 | | p4 |",+ "| aa | b1 | aa | p1 |",+ "+----+-----------+-----+-----------+",+ ];+ assert_batches_sorted_eq!(expected, &batches);+}++/// A parent filter over one side's join keys is transferred to the other side,+/// rewritten over that side's key expressions, even when the key names differ.+/// Filters over non-key columns stay on their own side.+#[test]+fn test_hashjoin_parent_filter_transferred_across_join_keys() {+ let build_side_schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Utf8, false),+ Field::new("build_val", DataType::Utf8, false),+ ]));+ let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))+ .with_support(true)+ .build();++ let probe_side_schema = Arc::new(Schema::new(vec![+ Field::new("pid", DataType::Utf8, false),+ Field::new("probe_val", DataType::Utf8, false),+ ]));+ let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))+ .with_support(true)+ .build();++ let on = vec![(+ col("id", &build_side_schema).unwrap(),+ col("pid", &probe_side_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ build_scan,+ probe_scan,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );++ let join_schema = join.schema();++ let build_key_filter = col_lit_predicate("id", "aa", &join_schema);+ let probe_key_filter = col_lit_predicate("pid", "ab", &join_schema);+ let build_val_filter = col_lit_predicate("build_val", "x", &join_schema);++ let filter =+ Arc::new(FilterExec::try_new(build_key_filter, Arc::clone(&join) as _).unwrap());+ let filter = Arc::new(FilterExec::try_new(probe_key_filter, filter).unwrap());+ let plan = Arc::new(FilterExec::try_new(build_val_filter, filter).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: build_val@1 = x+ - FilterExec: pid@2 = ab+ - FilterExec: id@0 = aa+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(id@0, pid@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, build_val], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[pid, probe_val], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(id@0, pid@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[id, build_val], file_type=test, pushdown_supported=true, predicate=id@0 = aa AND id@0 = ab AND build_val@1 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[pid, probe_val], file_type=test, pushdown_supported=true, predicate=pid@0 = aa AND pid@0 = ab+ "+ );+}++/// The non-output side of a semi join receives key filters through the same+/// transfer, so differently named keys work too.+#[test]+fn test_hashjoin_parent_filter_transfer_semi_join_different_key_names() {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Utf8, false),+ Field::new("v", DataType::Utf8, false),+ ]));+ let left_scan = TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build();++ let right_schema = Arc::new(Schema::new(vec![+ Field::new("w", DataType::Utf8, false),+ Field::new("rk", DataType::Utf8, false),+ ]));+ let right_scan = TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build();++ let on = vec![(+ col("k", &left_schema).unwrap(),+ col("rk", &right_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ left_scan,+ right_scan,+ on,+ None,+ &JoinType::LeftSemi,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );++ let join_schema = join.schema();+ let key_filter = col_lit_predicate("k", "x", &join_schema);+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: k@0 = x+ - HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(k@0, rk@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[w, rk], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(k@0, rk@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=k@0 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[w, rk], file_type=test, pushdown_supported=true, predicate=rk@1 = x+ "+ );+}++/// `RightSemi` variant of the test above: the join outputs only the right+/// side, so a filter over its key reaches the left scan only by transfer.+#[test]+fn test_hashjoin_parent_filter_transfer_right_semi_join_different_key_names() {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Utf8, false),+ Field::new("v", DataType::Utf8, false),+ ]));+ let left_scan = TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build();++ let right_schema = Arc::new(Schema::new(vec![+ Field::new("w", DataType::Utf8, false),+ Field::new("rk", DataType::Utf8, false),+ ]));+ let right_scan = TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build();++ let on = vec![(+ col("k", &left_schema).unwrap(),+ col("rk", &right_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ left_scan,+ right_scan,+ on,+ None,+ &JoinType::RightSemi,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );++ let join_schema = join.schema();+ let key_filter = col_lit_predicate("rk", "x", &join_schema);+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: rk@1 = x+ - HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(k@0, rk@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[w, rk], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(k@0, rk@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=k@0 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[w, rk], file_type=test, pushdown_supported=true, predicate=rk@1 = x+ "+ );+}++/// Regression test for the name-based semi-join routing this transfer+/// replaced: the non-output side has a column with the key's *name* that is+/// not the key. The old code pushed the key filter to that column, so the+/// scan was pruned by the wrong column and the parent filter dropped. The+/// transfer rewrites the filter over the actual key instead.+#[test]+fn test_hashjoin_parent_filter_transfer_semi_join_key_name_shadowed_by_non_key() {+ // LeftSemi: the right side has a non-key `k` at index 2, the key is `j`.+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Utf8, false),+ Field::new("v", DataType::Utf8, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("j", DataType::Utf8, false),+ Field::new("w", DataType::Utf8, false),+ Field::new("k", DataType::Utf8, false),+ ]));+ let on = vec![(+ col("k", &left_schema).unwrap(),+ col("j", &right_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build(),+ TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build(),+ on,+ None,+ &JoinType::LeftSemi,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let key_filter = col_lit_predicate("k", "x", &join.schema());+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: k@0 = x+ - HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(k@0, j@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w, k], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(k@0, j@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=k@0 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w, k], file_type=test, pushdown_supported=true, predicate=j@0 = x+ "+ );++ // RightSemi mirror image: the left side has a non-key `k` at index 2.+ let on = vec![(+ col("j", &right_schema).unwrap(),+ col("k", &left_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build(),+ TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build(),+ on,+ None,+ &JoinType::RightSemi,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let key_filter = col_lit_predicate("k", "x", &join.schema());+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: k@0 = x+ - HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(j@0, k@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w, k], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(j@0, k@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w, k], file_type=test, pushdown_supported=true, predicate=j@0 = x+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=k@0 = x+ "+ );+}++/// A key column that appears in several `on` pairs is transferred over the+/// first pair. Any pair would be correct, since all of them are equal for a+/// matching row; this pins the documented choice.+#[test]+fn test_hashjoin_parent_filter_transfer_uses_first_on_pair() {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Utf8, false),+ Field::new("v", DataType::Utf8, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Utf8, false),+ Field::new("y", DataType::Utf8, false),+ ]));+ // ON k = x AND k = y+ let on = vec![+ (+ col("k", &left_schema).unwrap(),+ col("x", &right_schema).unwrap(),+ ),+ (+ col("k", &left_schema).unwrap(),+ col("y", &right_schema).unwrap(),+ ),+ ];+ let join = Arc::new(+ HashJoinExec::try_new(+ TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build(),+ TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build(),+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let key_filter = col_lit_predicate("k", "a", &join.schema());+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: k@0 = a+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(k@0, x@0), (k@0, y@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[x, y], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(k@0, x@0), (k@0, y@1)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=k@0 = a+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[x, y], file_type=test, pushdown_supported=true, predicate=x@0 = a+ "+ );+}++/// A transferred filter is rewritten over the other side's key *expression*,+/// here a `CAST`. The projection puts the right key at output index 0, the+/// same index as the left column inside the cast: the rewrite must not+/// descend into the substituted expression, or it would substitute again+/// without end.+#[test]+fn test_hashjoin_parent_filter_transfer_cast_key_with_projection() {+ let left_schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int32, false),+ Field::new("v", DataType::Utf8, false),+ ]));+ let right_schema = Arc::new(Schema::new(vec![+ Field::new("j", DataType::Int64, false),+ Field::new("w", DataType::Utf8, false),+ ]));+ let on = vec![(+ cast(+ col("k", &left_schema).unwrap(),+ &left_schema,+ DataType::Int64,+ )+ .unwrap(),+ col("j", &right_schema).unwrap(),+ )];+ let join = Arc::new(+ HashJoinExec::try_new(+ TestScanBuilder::new(Arc::clone(&left_schema))+ .with_support(true)+ .build(),+ TestScanBuilder::new(Arc::clone(&right_schema))+ .with_support(true)+ .build(),+ on,+ None,+ &JoinType::Inner,+ // Output only `j`, at index 0.+ Some(vec![2]),+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let key_filter = col_lit_predicate("j", 5i64, &join.schema());+ let plan = Arc::new(FilterExec::try_new(key_filter, join).unwrap())+ as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),+ @r"+ OptimizationTest:+ input:+ - FilterExec: j@0 = 5+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(k@0 AS Int64), j@0)], projection=[j@2]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w], file_type=test, pushdown_supported=true+ output:+ Ok:+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(k@0 AS Int64), j@0)], projection=[j@2]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[k, v], file_type=test, pushdown_supported=true, predicate=CAST(k@0 AS Int64) = 5+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[j, w], file_type=test, pushdown_supported=true, predicate=j@0 = 5+ "+ );+}++/// A join's dynamic filter over the build-side key of the join below it is+/// transferred across that join's keys onto its probe-side scan: the probe+/// scan is pruned by the keys of a table it is not joined with.+///+/// The lower build-side scan does not accept filters, so the lower join's own+/// dynamic filter still holds every `mid` key and prunes nothing: the rows the+/// bottom scan drops are dropped by the transferred filter alone.+#[tokio::test]+async fn test_hashjoin_dynamic_filter_transferred_through_nested_join() {+ // Upper build side: the two keys that survive.+ let top_schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Utf8, false)]));+ let top_scan = TestScanBuilder::new(Arc::clone(&top_schema))+ .with_support(true)+ .with_batches(vec![record_batch!(("t", Utf8, ["aa", "ab"])).unwrap()])+ .build();++ // Lower build side: joined with `top` on `m = t`. Rejects pushed filters,+ // so all four keys reach the lower join's hash table.+ let mid_schema = Arc::new(Schema::new(vec![+ Field::new("m", DataType::Utf8, false),+ Field::new("c", DataType::Float64, false),+ ]));+ let mid_scan = TestScanBuilder::new(Arc::clone(&mid_schema))+ .with_support(false)+ .with_batches(vec![+ record_batch!(+ ("m", Utf8, ["aa", "ab", "ac", "ad"]),+ ("c", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ])+ .build();++ // Lower probe side: joined with `mid` on `x = m`, never directly with `top`.+ let bottom_schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Utf8, false),+ Field::new("e", DataType::Float64, false),+ ]));+ let bottom_scan = TestScanBuilder::new(Arc::clone(&bottom_schema))+ .with_support(true)+ .with_batches(vec![+ record_batch!(+ ("x", Utf8, ["aa", "ab", "ac", "ad"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ])+ .build();++ let lower_join = Arc::new(+ HashJoinExec::try_new(+ mid_scan,+ Arc::clone(&bottom_scan),+ vec![(+ col("m", &mid_schema).unwrap(),+ col("x", &bottom_schema).unwrap(),+ )],+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let lower_schema = lower_join.schema();+ let upper_join = Arc::new(+ HashJoinExec::try_new(+ top_scan,+ lower_join,+ vec![(+ col("t", &top_schema).unwrap(),+ col("m", &lower_schema).unwrap(),+ )],+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );+ let upper_schema = upper_join.schema();+ let plan = Arc::new(SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr::new(+ col("x", &upper_schema).unwrap(),+ SortOptions::new(false, false),+ )])+ .unwrap(),+ upper_join,+ )) as Arc<dyn ExecutionPlan>;++ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[x@3 ASC NULLS LAST], preserve_partitioning=[false]+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t@0, m@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[t], file_type=test, pushdown_supported=true+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(m@0, x@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[m, c], file_type=test, pushdown_supported=false+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[x, e], file_type=test, pushdown_supported=true+ output:+ Ok:+ - SortExec: expr=[x@3 ASC NULLS LAST], preserve_partitioning=[false]+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t@0, m@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[t], file_type=test, pushdown_supported=true+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(m@0, x@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[m, c], file_type=test, pushdown_supported=false+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[x, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ]+ "+ );++ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_dynamic_filter_pushdown = true;+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;++ // The bottom scan carries the lower join's own filter, which still lists+ // all four `mid` keys, and the upper join's filter rewritten over `x`.+ insta::assert_snapshot!(+ format_plan_for_test(&plan).to_string(),+ @r"+ - SortExec: expr=[x@3 ASC NULLS LAST], preserve_partitioning=[false]+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t@0, m@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[t], file_type=test, pushdown_supported=true+ - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(m@0, x@0)]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[m, c], file_type=test, pushdown_supported=false+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[x, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ x@0 >= aa AND x@0 <= ad AND x@0 IN (SET) ([aa, ab, ac, ad]) ] AND DynamicFilter [ x@0 >= aa AND x@0 <= ab AND x@0 IN (SET) ([aa, ab]) ]+ "+ );++ // The lower join's own filter lets all four `bottom` rows through; the+ // transferred filter from `top` prunes them to two before the join.+ let bottom_scan_metrics = bottom_scan.metrics().unwrap();+ assert_eq!(bottom_scan_metrics.output_rows().unwrap(), 2);++ insta::assert_snapshot!(+ format!("{}", pretty_format_batches(&batches).unwrap()),+ @r"+ +----+----+-----+----+-----++ | t | m | c | x | e |+ +----+----+-----+----+-----++ | aa | aa | 1.0 | aa | 1.0 |+ | ab | ab | 2.0 | ab | 2.0 |+ +----+----+-----+----+-----++ ",+ );+}+#[test]fn test_filter_pushdown_through_union() {let scan1 = TestScanBuilder::new(schema()).with_support(true).build();diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 088376bc34..c5fae3a65c 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -27,7 +27,7 @@ use crate::execution_plan::{};use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,- FilterPushdownPropagation,+ FilterPushdownPropagation, PushedDownPredicate,};use crate::joins::Map;use crate::joins::array_map::ArrayMap;@@ -76,7 +76,7 @@ use arrow::record_batch::RecordBatch;use arrow::util::bit_util;use arrow_schema::{DataType, Schema};use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::utils::memory::{RecordBatchMemoryCounter, estimate_memory_size};use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,@@ -979,6 +979,73 @@ impl HashJoinExec {Arc::new(DynamicFilterPhysicalExpr::new(right_keys, lit(true)))}+ /// Join types whose output rows all carry a matching key on both sides.+ ///+ /// For these a parent filter over one side's join keys can be transferred+ /// to the other side's input: an input row that fails the transferred+ /// filter can only pair with rows that fail the original, so pruning it+ /// changes nothing, and once the transferred filter is applied exactly on+ /// one side every output row satisfies the original. Outer, anti and mark+ /// joins also emit unmatched rows, whose key on the other side is absent,+ /// so the transferred filter is not exact for them.+ fn supports_key_transfer(join_type: JoinType) -> bool {+ matches!(+ join_type,+ JoinType::Inner | JoinType::LeftSemi | JoinType::RightSemi+ )+ }++ /// Maps each output column that is a plain `Column` join key on one side+ /// to the key expression on the other side, as `(to_right, to_left)`.+ ///+ /// `column_indices` are the (projected) output columns of this join. A key+ /// column that appears in several `on` pairs maps to the first of them.+ fn key_transfer_maps(+ &self,+ column_indices: &[ColumnIndex],+ ) -> (KeyTransferMap, KeyTransferMap) {+ // A transferred filter compares the other side's key with literals+ // typed for this side's key. The planner coerces both keys to one+ // type, and `try_new` does not check it, so make the assumption+ // explicit here.+ debug_assert!(+ self.on.iter().all(|(left_key, right_key)| {+ left_key.data_type(&self.left.schema()).ok()+ == right_key.data_type(&self.right.schema()).ok()+ }),+ "join key data types differ: {:?}",+ self.on+ );+ let mut to_right = HashMap::new();+ let mut to_left = HashMap::new();+ for (output_idx, ci) in column_indices.iter().enumerate() {+ let (map, other_key) = match ci.side {+ JoinSide::Left => (+ &mut to_right,+ self.on+ .iter()+ .find(|(left_key, _)| is_column_at(left_key, ci.index))+ .map(|(_, right_key)| right_key),+ ),+ JoinSide::Right => (+ &mut to_left,+ self.on+ .iter()+ .find(|(_, right_key)| is_column_at(right_key, ci.index))+ .map(|(left_key, _)| left_key),+ ),+ // Only mark joins produce mark columns, and+ // `supports_key_transfer` excludes them; this arm is here for+ // exhaustiveness.+ JoinSide::None => continue,+ };+ if let Some(other_key) = other_key {+ map.insert(output_idx, Arc::clone(other_key));+ }+ }+ (to_right, to_left)+ }+fn allow_join_dynamic_filter_pushdown(&self, config: &ConfigOptions) -> bool {let (_, probe_preserved) = self.join_type.on_lr_is_preserved();if !probe_preserved || !config.optimizer.enable_join_dynamic_filter_pushdown {@@ -1849,58 +1916,40 @@ impl ExecutionPlan for HashJoinExec {}}- // Semi joins also allow filters on output join keys to reach the- // non-output side. Map to the paired key's position, which need not- // have the same name. Only direct column pairs can be remapped this way.- // Anti joins cannot do this: their emitted rows have no matching key.- if matches!(self.join_type, JoinType::LeftSemi | JoinType::RightSemi) {- let key_mapping: HashMap<usize, usize> = self- .on- .iter()- .filter_map(|(left_key, right_key)| {- let left = left_key.downcast_ref::<Column>()?;- let right = right_key.downcast_ref::<Column>()?;- Some(match self.join_type {- JoinType::LeftSemi => (left.index(), right.index()),- _ => (right.index(), left.index()),- })- })- .collect();- let (output_side, other_mapping) = match self.join_type {- JoinType::LeftSemi => (JoinSide::Left, &mut right_mapping),- _ => (JoinSide::Right, &mut left_mapping),- };- let emitted = column_indices- .iter()- .enumerate()- .filter(|(_, ci)| ci.side == output_side);- for (output_idx, ci) in emitted {- if let Some(&input_idx) = key_mapping.get(&ci.index) {- other_mapping.insert(output_idx, input_idx);- }- }- }-- let left_child = if left_preserved {- ChildFilterDescription::from_child_with_column_mapping(- &parent_filters,- left_mapping,- self.left(),- )?+ // Transfer filters across the equi-join keys: a parent filter over one+ // side's join-key columns holds for every matching row of the other+ // side too, so it is also pushed there, rewritten over that side's key+ // expressions. This is how a dynamic filter from a join above reaches+ // the scans on both sides of this join, and how a semi join prunes its+ // non-output side. Like the plain column routing, a transfer only+ // targets a side that `lr_is_preserved` permits.+ let (to_right, to_left) = if Self::supports_key_transfer(self.join_type) {+ self.key_transfer_maps(&column_indices)} else {- ChildFilterDescription::all_unsupported(&parent_filters)+ Default::default()};-- let mut right_child = if right_preserved {- ChildFilterDescription::from_child_with_column_mapping(+ let describe_child = |preserved: bool,+ column_mapping: HashMap<usize, usize>,+ key_map: &KeyTransferMap,+ child: &Arc<dyn ExecutionPlan>|+ -> Result<ChildFilterDescription> {+ if !preserved {+ return Ok(ChildFilterDescription::all_unsupported(&parent_filters));+ }+ let mut description = ChildFilterDescription::from_child_with_column_mapping(&parent_filters,- right_mapping,- self.right(),- )?- } else {- ChildFilterDescription::all_unsupported(&parent_filters)+ column_mapping,+ child,+ )?;+ transfer_key_filters(&parent_filters, key_map, &mut description)?;+ Ok(description)};+ let left_child =+ describe_child(left_preserved, left_mapping, &to_left, self.left())?;+ let mut right_child =+ describe_child(right_preserved, right_mapping, &to_right, self.right())?;+// Add dynamic filters in Post phase if enabled. Skip when this join// already carries a dynamic filter from a previous pass — the shared// `Arc<DynamicFilterPhysicalExpr>` is still wired into the probe-side@@ -2510,6 +2559,73 @@ mod proto_tests {}}+/// Output column index of a join, mapped to the equivalent join-key expression+/// on the other side of the join (in that side's input schema).+type KeyTransferMap = HashMap<usize, PhysicalExprRef>;++fn is_column_at(expr: &PhysicalExprRef, index: usize) -> bool {+ expr.downcast_ref::<Column>()+ .is_some_and(|column| column.index() == index)+}++/// Marks every parent filter whose columns are all join keys in `key_map` as+/// supported for `child`, rewritten over the other side's key expressions.+///+/// `key_map` only holds columns of the other side, so a filter it rewrites is+/// one the plain column analysis marked unsupported for `child`. A filter that+/// references any other column is left as that analysis routed it. A filter+/// with no columns comes back unchanged and was already accepted, so+/// rewriting it is a no-op.+fn transfer_key_filters(+ parent_filters: &[Arc<dyn PhysicalExpr>],+ key_map: &KeyTransferMap,+ child: &mut ChildFilterDescription,+) -> Result<()> {+ if key_map.is_empty() {+ return Ok(());+ }+ for (filter, pushed) in parent_filters.iter().zip(child.parent_filters.iter_mut()) {+ if let Some(transferred) = transfer_filter_across_keys(filter, key_map)? {+ *pushed = PushedDownPredicate::supported(transferred);+ }+ }+ Ok(())+}++/// Rewrites `filter` over the other side's join keys, or returns `None` when+/// it references a column that is not a transferable key.+///+/// A [`DynamicFilterPhysicalExpr`] comes out as a view sharing the original's+/// state with its key columns remapped, so it keeps tracking the build side.+fn transfer_filter_across_keys(+ filter: &Arc<dyn PhysicalExpr>,+ key_map: &KeyTransferMap,+) -> Result<Option<Arc<dyn PhysicalExpr>>> {+ let mut all_keys = true;+ let transformed = Arc::clone(filter).transform_down(|expr| {+ let Some(column) = expr.downcast_ref::<Column>() else {+ return Ok(Transformed::no(expr));+ };+ match key_map.get(&column.index()) {+ // The replacement is in the other side's input schema, so its+ // columns are not output indices of this join: `Jump` over it.+ // Descending would substitute again whenever the key column's+ // index is also an output index, e.g. `CAST(k@0 AS Int64)` for+ // output column 0, and never terminate.+ Some(other_key) => Ok(Transformed::new(+ Arc::clone(other_key),+ true,+ TreeNodeRecursion::Jump,+ )),+ None => {+ all_keys = false;+ Ok(Transformed::new(expr, false, TreeNodeRecursion::Stop))+ }+ }+ })?;+ Ok(all_keys.then_some(transformed.data))+}+/// Determines which sides of a join are "preserved" for filter pushdown.////// A preserved side means filters on that side's columns can be safely pushed@@ -2521,7 +2637,11 @@ fn lr_is_preserved(join_type: JoinType) -> (bool, bool) {JoinType::Left => (true, false),JoinType::Right => (false, true),JoinType::Full => (false, false),- // Callers restrict the non-output side of semi joins to join-key columns.+ // A semi join emits only matched rows, so pruning either input by a+ // filter its output satisfies is exact. The non-output side has no+ // output columns, so the column routing sends it nothing but+ // column-free filters; key filters reach it through the transfer in+ // `HashJoinExec::gather_filters_for_pushdown`.JoinType::LeftSemi | JoinType::RightSemi => (true, true),JoinType::LeftAnti | JoinType::LeftMark => (true, false),JoinType::RightAnti | JoinType::RightMark => (false, true),diff --git a/datafusion/sqllogictest/test_files/join_dynamic_filter_transfer.slt b/datafusion/sqllogictest/test_files/join_dynamic_filter_transfer.sltnew file mode 100644index 0000000000..04d3e4374f--- /dev/null+++ b/datafusion/sqllogictest/test_files/join_dynamic_filter_transfer.slt@@ -0,0 +1,135 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# A hash join's dynamic filter is transferred across the equi-join keys of the+# joins below it: a filter over one side's join key also holds for the other+# side's key, so it reaches the scans on both sides of that join.+#+# Here `dim` is the build side of the upper join and its keys filter `mid`+# directly. `fact` is only joined with `mid`, but through `mid.m_key = fact.f_key`+# the `dim` filter is rewritten over `f_key` and reaches the `fact` scan too.++statement ok+CREATE TABLE dim_src(d_key INT, d_val VARCHAR) AS VALUES+(1, 'one'),+(3, 'three');++statement ok+CREATE TABLE mid_src(m_key INT, m_c INT) AS VALUES+(1, 10),+(2, 20),+(3, 30),+(4, 40),+(5, 50);++statement ok+CREATE TABLE fact_src(f_key INT, f_e INT) AS VALUES+(1, 100),+(2, 200),+(3, 300),+(4, 400),+(5, 500),+(6, 600),+(7, 700),+(8, 800);++query I+COPY dim_src TO 'test_files/scratch/join_dynamic_filter_transfer/dim.parquet' STORED AS PARQUET;+----+2++query I+COPY mid_src TO 'test_files/scratch/join_dynamic_filter_transfer/mid.parquet' STORED AS PARQUET;+----+5++query I+COPY fact_src TO 'test_files/scratch/join_dynamic_filter_transfer/fact.parquet' STORED AS PARQUET;+----+8++statement ok+CREATE EXTERNAL TABLE dim(d_key INT, d_val VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/join_dynamic_filter_transfer/dim.parquet';++statement ok+CREATE EXTERNAL TABLE mid(m_key INT, m_c INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/join_dynamic_filter_transfer/mid.parquet';++statement ok+CREATE EXTERNAL TABLE fact(f_key INT, f_e INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/join_dynamic_filter_transfer/fact.parquet';++# The `fact` scan carries two dynamic filters: the lower join's own filter,+# built from `mid`, and the upper join's filter from `dim` transferred over+# `f_key`. The transferred one arrives with `dim`'s bounds and IN list even+# when `mid` is large enough that its own filter is a hash-table lookup, which+# cannot prune row groups.+query TT+EXPLAIN SELECT d.d_val, m.m_c, f.f_e+FROM mid m+JOIN fact f ON m.m_key = f.f_key+JOIN dim d ON d.d_key = m.m_key;+----+logical_plan+01)Projection: d.d_val, m.m_c, f.f_e+02)--Inner Join: m.m_key = d.d_key+03)----Projection: m.m_key, m.m_c, f.f_e+04)------Inner Join: m.m_key = f.f_key+05)--------SubqueryAlias: m+06)----------TableScan: mid projection=[m_key, m_c]+07)--------SubqueryAlias: f+08)----------TableScan: fact projection=[f_key, f_e]+09)----SubqueryAlias: d+10)------TableScan: dim projection=[d_key, d_val]+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_key@0, m_key@0)], projection=[d_val@1, m_c@3, f_e@4]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/join_dynamic_filter_transfer/dim.parquet]]}, projection=[d_key, d_val], file_type=parquet+03)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+04)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(m_key@0, f_key@0)], projection=[m_key@0, m_c@1, f_e@3]+05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/join_dynamic_filter_transfer/mid.parquet]]}, projection=[m_key, m_c], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible+06)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/join_dynamic_filter_transfer/fact.parquet]]}, projection=[f_key, f_e], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible++query TII rowsort+SELECT d.d_val, m.m_c, f.f_e+FROM mid m+JOIN fact f ON m.m_key = f.f_key+JOIN dim d ON d.d_key = m.m_key;+----+one 10 100+three 30 300++statement ok+DROP TABLE dim;++statement ok+DROP TABLE mid;++statement ok+DROP TABLE fact;++statement ok+DROP TABLE dim_src;++statement ok+DROP TABLE mid_src;++statement ok+DROP TABLE fact_src;