Apache DataFusion SQL Query Engine
Port filter_pushdown.rs async tests to sqllogictest (#21620)
Adrian Garcia Badaracco5c653bee5da6
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex a255c07545..d058e44a85 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -18,7 +18,7 @@use std::sync::{Arc, LazyLock};use arrow::{- array::{Float64Array, Int32Array, RecordBatch, StringArray, record_batch},+ array::record_batch,datatypes::{DataType, Field, Schema, SchemaRef},util::pretty::pretty_format_batches,};@@ -30,11 +30,10 @@ use datafusion::{PhysicalExpr,expressions::{BinaryExpr, Column, Literal},},- prelude::{ParquetReadOptions, SessionConfig, SessionContext},+ prelude::{SessionConfig, SessionContext},scalar::ScalarValue,};use datafusion_catalog::memory::DataSourceExec;-use datafusion_common::JoinType;use datafusion_common::config::ConfigOptions;use datafusion_datasource::{PartitionedFile, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder,@@ -42,19 +41,14 @@ use datafusion_datasource::{use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_expr::ScalarUDF;use datafusion_functions::math::random::RandomFunc;-use datafusion_functions_aggregate::{- count::count_udaf,- min_max::{max_udaf, min_udaf},-};+use datafusion_functions_aggregate::{count::count_udaf, min_max::min_udaf};use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, expressions::col};use datafusion_physical_expr::{- Partitioning, ScalarFunctionExpr,- aggregate::{AggregateExprBuilder, AggregateFunctionExpr},+ Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder,};use datafusion_physical_optimizer::{PhysicalOptimizerRule, filter_pushdown::FilterPushdown,};-use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion_physical_plan::{ExecutionPlan,aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},@@ -70,9 +64,7 @@ use super::pushdown_utils::{OptimizationTest, TestNode, TestScanBuilder, TestSource, format_plan_for_test,};use datafusion_physical_plan::union::UnionExec;-use futures::StreamExt;-use object_store::{ObjectStore, memory::InMemory};-use regex::Regex;+use object_store::memory::InMemory;#[test]fn test_pushdown_into_scan() {@@ -176,128 +168,11 @@ fn test_pushdown_into_scan_with_config_options() {);}-#[tokio::test]-async fn test_dynamic_filter_pushdown_through_hash_join_with_topk() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8View, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8View, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("d", Utf8, ["aa", "ab", "ac", "ad"]),- ("e", Utf8View, ["ba", "bb", "bc", "bd"]),- ("f", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("d", DataType::Utf8, false),- Field::new("e", DataType::Utf8View, false),- Field::new("f", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec- let on = vec![(- col("a", &build_side_schema).unwrap(),- col("d", &probe_side_schema).unwrap(),- )];- let join = Arc::new(- HashJoinExec::try_new(- build_scan,- probe_scan,- on,- None,- &JoinType::Inner,- None,- PartitionMode::Partitioned,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- );-- let join_schema = join.schema();-- // Finally let's add a SortExec on the outside to test pushdown of dynamic filters- let sort_expr =- PhysicalSortExpr::new(col("e", &join_schema).unwrap(), SortOptions::default());- let plan = Arc::new(- SortExec::new(LexOrdering::new(vec![sort_expr]).unwrap(), join)- .with_fetch(Some(2)),- ) as Arc<dyn ExecutionPlan>;-- let mut config = ConfigOptions::default();- config.optimizer.enable_dynamic_filter_pushdown = true;- config.execution.parquet.pushdown_filters = true;-- // Apply the FilterPushdown optimizer rule- let plan = FilterPushdown::new_post_optimization()- .optimize(Arc::clone(&plan), &config)- .unwrap();-- // Test that filters are pushed down correctly to each side of the join- insta::assert_snapshot!(- format_plan_for_test(&plan),- @r"- - SortExec: TopK(fetch=2), expr=[e@4 ASC], preserve_partitioning=[false]- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ]- "- );-- // Put some data through the plan to check that the filter is updated to reflect the TopK state- let session_ctx = SessionContext::new_with_config(SessionConfig::new());- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- // Iterate one batch- stream.next().await.unwrap().unwrap();-- // Test that filters are pushed down correctly to each side of the join- // NOTE: We dropped the CASE expression here because we now optimize that away if there's only 1 partition- insta::assert_snapshot!(- format_plan_for_test(&plan),- @r"- - SortExec: TopK(fetch=2), expr=[e@4 ASC], preserve_partitioning=[false], filter=[e@4 IS NULL OR e@4 < bb]- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ d@0 >= aa AND d@0 <= ab AND d@0 IN (SET) ([aa, ab]) ] AND DynamicFilter [ e@1 IS NULL OR e@1 < bb ]- "- );-}--// Test both static and dynamic filter pushdown in HashJoinExec.-// Note that static filter pushdown is rare: it should have already happened in the logical optimizer phase.-// However users may manually construct plans that could result in a FilterExec -> HashJoinExec -> Scan setup.-// Dynamic filters arise in cases such as nested inner joins or TopK -> HashJoinExec -> Scan setups.+// Inner-join part is covered by push_down_filter_parquet.slt::test_hashjoin_parent_filter_pushdown.+// The Left-join part stays in Rust: SQL's outer-join-elimination rewrites+// `LEFT JOIN ... WHERE <probe-side-null-rejecting>` into an INNER JOIN+// before physical filter pushdown runs, so the preserved-vs-non-preserved+// distinction this test exercises is not reachable via SQL.#[tokio::test]async fn test_static_filter_pushdown_through_hash_join() {use datafusion_common::JoinType;@@ -926,175 +801,11 @@ fn test_node_handles_child_pushdown_result() {);}-#[tokio::test]-async fn test_topk_dynamic_filter_pushdown() {- let batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["bd", "bc"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- record_batch!(- ("a", Utf8, ["ac", "ad"]),- ("b", Utf8, ["bb", "ba"]),- ("c", Float64, [2.0, 1.0])- )- .unwrap(),- ];- let scan = TestScanBuilder::new(schema())- .with_support(true)- .with_batches(batches)- .build();- let plan = Arc::new(- SortExec::new(- LexOrdering::new(vec![PhysicalSortExpr::new(- col("b", &schema()).unwrap(),- SortOptions::new(true, false), // descending, nulls_first- )])- .unwrap(),- Arc::clone(&scan),- )- .with_fetch(Some(1)),- ) as Arc<dyn ExecutionPlan>;-- // expect the predicate to be pushed down into the DataSource- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- "- );-- // Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK state- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();- let config = SessionConfig::new().with_batch_size(2);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- // Iterate one batch- stream.next().await.unwrap().unwrap();- // Now check what our filter looks like- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false], filter=[b@1 > bd]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ b@1 > bd ]- "- );-}--#[tokio::test]-async fn test_topk_dynamic_filter_pushdown_multi_column_sort() {- let batches = vec![- // We are going to do ORDER BY b ASC NULLS LAST, a DESC- // And we put the values in such a way that the first batch will fill the TopK- // and we skip the second batch.- record_batch!(- ("a", Utf8, ["ac", "ad"]),- ("b", Utf8, ["bb", "ba"]),- ("c", Float64, [2.0, 1.0])- )- .unwrap(),- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["bc", "bd"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- ];- let scan = TestScanBuilder::new(schema())- .with_support(true)- .with_batches(batches)- .build();- let plan = Arc::new(- SortExec::new(- LexOrdering::new(vec![- PhysicalSortExpr::new(- col("b", &schema()).unwrap(),- SortOptions::default().asc().nulls_last(),- ),- PhysicalSortExpr::new(- col("a", &schema()).unwrap(),- SortOptions::default().desc().nulls_first(),- ),- ])- .unwrap(),- Arc::clone(&scan),- )- .with_fetch(Some(2)),- ) as Arc<dyn ExecutionPlan>;-- // expect the predicate to be pushed down into the DataSource- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- "- );-- // Actually apply the optimization to the plan and put some data through it to check that the filter is updated to reflect the TopK state- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();- let config = SessionConfig::new().with_batch_size(2);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- // Iterate one batch- let res = stream.next().await.unwrap().unwrap();- #[rustfmt::skip]- let expected = [- "+----+----+-----+",- "| a | b | c |",- "+----+----+-----+",- "| ad | ba | 1.0 |",- "| ac | bb | 2.0 |",- "+----+----+-----+",- ];- assert_batches_eq!(expected, &[res]);- // Now check what our filter looks like- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false], filter=[b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ]- "- );- // There should be no more batches- assert!(stream.next().await.is_none());-}-+// Not portable to sqllogictest: requires manually constructing+// `SortExec(CoalescePartitionsExec(scan))`. A SQL `ORDER BY ... LIMIT` over a+// multi-partition scan plans as `SortPreservingMergeExec(SortExec(scan))`+// instead, so the filter-through-coalesce path this test exercises is not+// reachable via SQL.#[tokio::test]async fn test_topk_filter_passes_through_coalesce_partitions() {// Create multiple batches for different partitions@@ -1163,129 +874,12 @@ async fn test_topk_filter_passes_through_coalesce_partitions() {);}+// Not portable to sqllogictest: this test pins `PartitionMode::Partitioned`+// by hand-wiring `RepartitionExec(Hash, 12)` on both join sides. A SQL+// INNER JOIN over small parquet inputs plans as `CollectLeft`, so the+// per-partition CASE filter this test exercises is not reachable via SQL.#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec with dynamic filter- let on = vec![- (- col("a", &build_side_schema).unwrap(),- col("a", &probe_side_schema).unwrap(),- ),- (- col("b", &build_side_schema).unwrap(),- col("b", &probe_side_schema).unwrap(),- ),- ];- let plan = Arc::new(- HashJoinExec::try_new(- build_scan,- probe_scan,- on,- None,- &JoinType::Inner,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- ) as Arc<dyn ExecutionPlan>;-- // expect the predicate to be pushed down into the probe side DataSource- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true- output:- Ok:- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- ",- );-- // Actually apply the optimization to the plan and execute to see the filter in action- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();-- // Test for https://github.com/apache/datafusion/pull/17371: dynamic filter linking survives `with_new_children`- let children = plan.children().into_iter().map(Arc::clone).collect();- let plan = plan.with_new_children(children).unwrap();-- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- // Iterate one batch- stream.next().await.unwrap().unwrap();-- // Now check what our filter looks like- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ]- "- );-}--#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {+async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {use datafusion_common::JoinType;use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};@@ -1533,6 +1127,12 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {);}+// Not portable to sqllogictest: this test specifically pins a+// `RepartitionExec(Hash, 12)` between `HashJoinExec(CollectLeft)` and the+// probe-side scan to verify the dynamic filter link survives that boundary+// (regression for #17451). The same CollectLeft filter content and+// pushdown counters are already covered by the simpler slt port+// (push_down_filter_parquet.slt::test_hashjoin_dynamic_filter_pushdown).#[tokio::test]async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {use datafusion_common::JoinType;@@ -1705,265 +1305,6 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {);}-#[tokio::test]-async fn test_nested_hashjoin_dynamic_filter_pushdown() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create test data for three tables: t1, t2, t3- // t1: small table with limited values (will be build side of outer join)- let t1_batches = vec![- record_batch!(("a", Utf8, ["aa", "ab"]), ("x", Float64, [1.0, 2.0])).unwrap(),- ];- let t1_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("x", DataType::Float64, false),- ]));- let t1_scan = TestScanBuilder::new(Arc::clone(&t1_schema))- .with_support(true)- .with_batches(t1_batches)- .build();-- // t2: larger table (will be probe side of inner join, build side of outer join)- let t2_batches = vec![- record_batch!(- ("b", Utf8, ["aa", "ab", "ac", "ad", "ae"]),- ("c", Utf8, ["ca", "cb", "cc", "cd", "ce"]),- ("y", Float64, [1.0, 2.0, 3.0, 4.0, 5.0])- )- .unwrap(),- ];- let t2_schema = Arc::new(Schema::new(vec![- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Utf8, false),- Field::new("y", DataType::Float64, false),- ]));- let t2_scan = TestScanBuilder::new(Arc::clone(&t2_schema))- .with_support(true)- .with_batches(t2_batches)- .build();-- // t3: largest table (will be probe side of inner join)- let t3_batches = vec![- record_batch!(- ("d", Utf8, ["ca", "cb", "cc", "cd", "ce", "cf", "cg", "ch"]),- ("z", Float64, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])- )- .unwrap(),- ];- let t3_schema = Arc::new(Schema::new(vec![- Field::new("d", DataType::Utf8, false),- Field::new("z", DataType::Float64, false),- ]));- let t3_scan = TestScanBuilder::new(Arc::clone(&t3_schema))- .with_support(true)- .with_batches(t3_batches)- .build();-- // Create nested join structure:- // Join (t1.a = t2.b)- // / \- // t1 Join(t2.c = t3.d)- // / \- // t2 t3-- // First create inner join: t2.c = t3.d- let inner_join_on =- vec![(col("c", &t2_schema).unwrap(), col("d", &t3_schema).unwrap())];- let inner_join = Arc::new(- HashJoinExec::try_new(- t2_scan,- t3_scan,- inner_join_on,- None,- &JoinType::Inner,- None,- PartitionMode::Partitioned,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- );-- // Then create outer join: t1.a = t2.b (from inner join result)- let outer_join_on = vec![(- col("a", &t1_schema).unwrap(),- col("b", &inner_join.schema()).unwrap(),- )];- let outer_join = Arc::new(- HashJoinExec::try_new(- t1_scan,- inner_join as Arc<dyn ExecutionPlan>,- outer_join_on,- None,- &JoinType::Inner,- None,- PartitionMode::Partitioned,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- ) as Arc<dyn ExecutionPlan>;-- // Test that dynamic filters are pushed down correctly through nested joins- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&outer_join), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, x], file_type=test, pushdown_supported=true- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@1, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[b, c, y], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, z], file_type=test, pushdown_supported=true- output:- Ok:- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, x], file_type=test, pushdown_supported=true- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@1, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[b, c, y], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, z], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- ",- );-- // Execute the plan to verify the dynamic filters are properly updated- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(outer_join, &config)- .unwrap();- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- // Execute to populate the dynamic filters- stream.next().await.unwrap().unwrap();-- // Verify that both the inner and outer join have updated dynamic filters- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, x], file_type=test, pushdown_supported=true- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@1, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[b, c, y], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ b@0 >= aa AND b@0 <= ab AND b@0 IN (SET) ([aa, ab]) ]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, z], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND d@0 IN (SET) ([ca, cb]) ]- "- );-}--#[tokio::test]-async fn test_hashjoin_parent_filter_pushdown() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("d", Utf8, ["aa", "ab", "ac", "ad"]),- ("e", Utf8, ["ba", "bb", "bc", "bd"]),- ("f", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("d", DataType::Utf8, false),- Field::new("e", DataType::Utf8, false),- Field::new("f", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec- let on = vec![(- col("a", &build_side_schema).unwrap(),- col("d", &probe_side_schema).unwrap(),- )];- let join = Arc::new(- HashJoinExec::try_new(- build_scan,- probe_scan,- on,- None,- &JoinType::Inner,- None,- PartitionMode::Partitioned,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- );-- // Create filters that can be pushed down to different sides- // We need to create filters in the context of the join output schema- let join_schema = join.schema();-- // Filter on build side column: a = 'aa'- let left_filter = col_lit_predicate("a", "aa", &join_schema);- // Filter on probe side column: e = 'ba'- let right_filter = col_lit_predicate("e", "ba", &join_schema);- // Filter that references both sides: a = d (should not be pushed down)- let cross_filter = Arc::new(BinaryExpr::new(- col("a", &join_schema).unwrap(),- Operator::Eq,- col("d", &join_schema).unwrap(),- )) as Arc<dyn PhysicalExpr>;-- let filter =- Arc::new(FilterExec::try_new(left_filter, Arc::clone(&join) as _).unwrap());- let filter = Arc::new(FilterExec::try_new(right_filter, filter).unwrap());- let plan = Arc::new(FilterExec::try_new(cross_filter, filter).unwrap())- as Arc<dyn ExecutionPlan>;-- // Test that filters are pushed down correctly to each side of the join- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),- @r"- OptimizationTest:- input:- - FilterExec: a@0 = d@3- - FilterExec: e@4 = ba- - FilterExec: a@0 = aa- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true- output:- Ok:- - FilterExec: a@0 = d@3- - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, d@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = aa- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[d, e, f], file_type=test, pushdown_supported=true, predicate=e@1 = ba- "- );-}-#[test]fn test_hashjoin_parent_filter_pushdown_same_column_names() {use datafusion_common::JoinType;@@ -2176,64 +1517,6 @@ fn test_hashjoin_parent_filter_pushdown_semi_anti_join() {);}-/// Integration test for dynamic filter pushdown with TopK.-/// We use an integration test because there are complex interactions in the optimizer rules-/// that the unit tests applying a single optimizer rule do not cover.-#[tokio::test]-async fn test_topk_dynamic_filter_pushdown_integration() {- let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;- let mut cfg = SessionConfig::new();- cfg.options_mut().execution.parquet.pushdown_filters = true;- cfg.options_mut().execution.parquet.max_row_group_size = 128;- let ctx = SessionContext::new_with_config(cfg);- ctx.register_object_store(- ObjectStoreUrl::parse("memory://").unwrap().as_ref(),- Arc::clone(&store),- );- ctx.sql(- r"-COPY (- SELECT 1372708800 + value AS t- FROM generate_series(0, 99999)- ORDER BY t- ) TO 'memory:///1.parquet'-STORED AS PARQUET;- ",- )- .await- .unwrap()- .collect()- .await- .unwrap();-- // Register the file with the context- ctx.register_parquet(- "topk_pushdown",- "memory:///1.parquet",- ParquetReadOptions::default(),- )- .await- .unwrap();-- // Create a TopK query that will use dynamic filter pushdown- // Note that we use t * t as the order by expression to avoid- // the order pushdown optimizer from optimizing away the TopK.- let df = ctx- .sql(r"EXPLAIN ANALYZE SELECT t FROM topk_pushdown ORDER BY t * t LIMIT 10;")- .await- .unwrap();- let batches = df.collect().await.unwrap();- let explain = format!("{}", pretty_format_batches(&batches).unwrap());-- assert!(explain.contains("output_rows=128")); // Read 1 row group- assert!(explain.contains("t@0 < 1884329474306198481")); // Dynamic filter was applied- assert!(- explain.contains("pushdown_rows_matched=128, pushdown_rows_pruned=99.87 K"),- "{explain}"- );- // Pushdown pruned most rows-}-#[test]fn test_filter_pushdown_through_union() {let scan1 = TestScanBuilder::new(schema()).with_support(true).build();@@ -2450,233 +1733,11 @@ fn schema() -> SchemaRef {Arc::clone(&TEST_SCHEMA)}-struct ProjectionDynFilterTestCase {- schema: SchemaRef,- batches: Vec<RecordBatch>,- projection: Vec<(Arc<dyn PhysicalExpr>, String)>,- sort_expr: PhysicalSortExpr,- expected_plans: Vec<String>,-}--async fn run_projection_dyn_filter_case(case: ProjectionDynFilterTestCase) {- let ProjectionDynFilterTestCase {- schema,- batches,- projection,- sort_expr,- expected_plans,- } = case;-- let scan = TestScanBuilder::new(Arc::clone(&schema))- .with_support(true)- .with_batches(batches)- .build();-- let projection_exec = Arc::new(ProjectionExec::try_new(projection, scan).unwrap());-- let sort = Arc::new(- SortExec::new(LexOrdering::new(vec![sort_expr]).unwrap(), projection_exec)- .with_fetch(Some(2)),- ) as Arc<dyn ExecutionPlan>;-- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;-- let optimized_plan = FilterPushdown::new_post_optimization()- .optimize(Arc::clone(&sort), &config)- .unwrap();-- pretty_assertions::assert_eq!(- format_plan_for_test(&optimized_plan).trim(),- expected_plans[0].trim()- );-- let config = SessionConfig::new().with_batch_size(2);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = optimized_plan.execute(0, Arc::clone(&task_ctx)).unwrap();- for (idx, expected_plan) in expected_plans.iter().enumerate().skip(1) {- stream.next().await.unwrap().unwrap();- let formatted_plan = format_plan_for_test(&optimized_plan);- pretty_assertions::assert_eq!(- formatted_plan.trim(),- expected_plan.trim(),- "Mismatch at iteration {}",- idx- );- }-}--#[tokio::test]-async fn test_topk_with_projection_transformation_on_dyn_filter() {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let simple_abc = vec![- record_batch!(- ("a", Int32, [1, 2, 3]),- ("b", Utf8, ["x", "y", "z"]),- ("c", Float64, [1.0, 2.0, 3.0])- )- .unwrap(),- ];-- // Case 1: Reordering [b, a]- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![- (col("b", &schema).unwrap(), "b".to_string()),- (col("a", &schema).unwrap(), "a".to_string()),- ],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("a", 1)),- SortOptions::default(),- ),- expected_plans: vec![-r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[b@1 as b, a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),-r#" - SortExec: TopK(fetch=2), expr=[a@1 ASC], preserve_partitioning=[false], filter=[a@1 IS NULL OR a@1 < 2]- - ProjectionExec: expr=[b@1 as b, a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string()]- })- .await;-- // Case 2: Pruning [a]- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![(col("a", &schema).unwrap(), "a".to_string())],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("a", 0)),- SortOptions::default(),- ),- expected_plans: vec![- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]- - ProjectionExec: expr=[a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),- ],- })- .await;-- // Case 3: Identity [a, b]- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![- (col("a", &schema).unwrap(), "a".to_string()),- (col("b", &schema).unwrap(), "b".to_string()),- ],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("a", 0)),- SortOptions::default(),- ),- expected_plans: vec![- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 2]- - ProjectionExec: expr=[a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),- ],- })- .await;-- // Case 4: Expressions [a + 1, b]- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![- (- Arc::new(BinaryExpr::new(- col("a", &schema).unwrap(),- Operator::Plus,- Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),- )),- "a_plus_1".to_string(),- ),- (col("b", &schema).unwrap(), "b".to_string()),- ],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("a_plus_1", 0)),- SortOptions::default(),- ),- expected_plans: vec![- r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),- r#" - SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC], preserve_partitioning=[false], filter=[a_plus_1@0 IS NULL OR a_plus_1@0 < 3]- - ProjectionExec: expr=[a@0 + 1 as a_plus_1, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),- ],- })- .await;-- // Case 5: [a as b, b as a] (swapped columns)- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![- (col("a", &schema).unwrap(), "b".to_string()),- (col("b", &schema).unwrap(), "a".to_string()),- ],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("b", 0)),- SortOptions::default(),- ),- expected_plans: vec![- r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[a@0 as b, b@1 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),- r#" - SortExec: TopK(fetch=2), expr=[b@0 ASC], preserve_partitioning=[false], filter=[b@0 IS NULL OR b@0 < 2]- - ProjectionExec: expr=[a@0 as b, b@1 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 IS NULL OR a@0 < 2 ]"#.to_string(),- ],- })- .await;-- // Case 6: Confusing expr [a + 1 as a, b]- run_projection_dyn_filter_case(ProjectionDynFilterTestCase {- schema: Arc::clone(&schema),- batches: simple_abc.clone(),- projection: vec![- (- Arc::new(BinaryExpr::new(- col("a", &schema).unwrap(),- Operator::Plus,- Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),- )),- "a".to_string(),- ),- (col("b", &schema).unwrap(), "b".to_string()),- ],- sort_expr: PhysicalSortExpr::new(- Arc::new(Column::new("a", 0)),- SortOptions::default(),- ),- expected_plans: vec![- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false]- - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]"#.to_string(),- r#" - SortExec: TopK(fetch=2), expr=[a@0 ASC], preserve_partitioning=[false], filter=[a@0 IS NULL OR a@0 < 3]- - ProjectionExec: expr=[a@0 + 1 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 + 1 IS NULL OR a@0 + 1 < 3 ]"#.to_string(),- ],- })- .await;-}+// test_topk_with_projection_transformation_on_dyn_filter has been ported+// to datafusion/sqllogictest/test_files/push_down_filter_parquet.slt; see+// `topk_proj` fixture for the 4 representative cases (reorder, prune,+// expression, alias shadowing). The `run_projection_dyn_filter_case`+// harness was removed along with it./// Returns a predicate that is a binary expression col = litfn col_lit_predicate(@@ -2693,382 +1754,12 @@ fn col_lit_predicate(}// ==== Aggregate Dynamic Filter tests ====--// ---- Test Utilities -----struct AggregateDynFilterCase<'a> {- schema: SchemaRef,- batches: Vec<RecordBatch>,- aggr_exprs: Vec<AggregateFunctionExpr>,- expected_before: Option<&'a str>,- expected_after: Option<&'a str>,- scan_support: bool,-}--async fn run_aggregate_dyn_filter_case(case: AggregateDynFilterCase<'_>) {- let AggregateDynFilterCase {- schema,- batches,- aggr_exprs,- expected_before,- expected_after,- scan_support,- } = case;-- let scan = TestScanBuilder::new(Arc::clone(&schema))- .with_support(scan_support)- .with_batches(batches)- .build();-- let aggr_exprs: Vec<_> = aggr_exprs- .into_iter()- .map(|expr| Arc::new(expr) as Arc<AggregateFunctionExpr>)- .collect();- let aggr_len = aggr_exprs.len();-- let plan: Arc<dyn ExecutionPlan> = Arc::new(- AggregateExec::try_new(- AggregateMode::Partial,- PhysicalGroupBy::new_single(vec![]),- aggr_exprs,- vec![None; aggr_len],- scan,- Arc::clone(&schema),- )- .unwrap(),- );-- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;-- let optimized = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();-- let before = format_plan_for_test(&optimized);- if let Some(expected) = expected_before {- assert!(- before.contains(expected),- "expected `{expected}` before execution, got: {before}"- );- } else {- assert!(- !before.contains("DynamicFilter ["),- "dynamic filter unexpectedly present before execution: {before}"- );- }-- let session_ctx = SessionContext::new();- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let task_ctx = session_ctx.state().task_ctx();- let mut stream = optimized.execute(0, Arc::clone(&task_ctx)).unwrap();- let _ = stream.next().await.transpose().unwrap();-- let after = format_plan_for_test(&optimized);- if let Some(expected) = expected_after {- assert!(- after.contains(expected),- "expected `{expected}` after execution, got: {after}"- );- } else {- assert!(- !after.contains("DynamicFilter ["),- "dynamic filter unexpectedly present after execution: {after}"- );- }-}--// ---- Test Cases -----// Cases covered below:-// 1. `min(a)` and `max(a)` baseline.-// 2. Unsupported expression input (`min(a+1)`).-// 3. Multiple supported columns (same column vs different columns).-// 4. Mixed supported + unsupported aggregates.-// 5. Entirely NULL input to surface current bound behavior.-// 6. End-to-end tests on parquet files--/// `MIN(a)`: able to pushdown dynamic filter-#[tokio::test]-async fn test_aggregate_dynamic_filter_min_simple() {- // Single min(a) showcases the base case.- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));- let batches = vec![record_batch!(("a", Int32, [5, 1, 3, 8])).unwrap()];-- let min_expr =- AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_expr],- expected_before: Some("DynamicFilter [ empty ]"),- expected_after: Some("DynamicFilter [ a@0 < 1 ]"),- scan_support: true,- })- .await;-}--/// `MAX(a)`: able to pushdown dynamic filter-#[tokio::test]-async fn test_aggregate_dynamic_filter_max_simple() {- // Single max(a) mirrors the base case on the upper bound.- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));- let batches = vec![record_batch!(("a", Int32, [5, 1, 3, 8])).unwrap()];-- let max_expr =- AggregateExprBuilder::new(max_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("max_a")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![max_expr],- expected_before: Some("DynamicFilter [ empty ]"),- expected_after: Some("DynamicFilter [ a@0 > 8 ]"),- scan_support: true,- })- .await;-}--/// `MIN(a+1)`: Can't pushdown dynamic filter-#[tokio::test]-async fn test_aggregate_dynamic_filter_min_expression_not_supported() {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));- let batches = vec![record_batch!(("a", Int32, [5, 1, 3, 8])).unwrap()];-- let expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- col("a", &schema).unwrap(),- Operator::Plus,- Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),- ));- let min_expr = AggregateExprBuilder::new(min_udaf(), vec![expr])- .schema(Arc::clone(&schema))- .alias("min_a_plus_one")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_expr],- expected_before: None,- expected_after: None,- scan_support: true,- })- .await;-}--/// `MIN(a), MAX(a)`: Pushdown dynamic filter like `(a<1) or (a>8)`-#[tokio::test]-async fn test_aggregate_dynamic_filter_min_max_same_column() {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));- let batches = vec![record_batch!(("a", Int32, [5, 1, 3, 8])).unwrap()];-- let min_expr =- AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .unwrap();- let max_expr =- AggregateExprBuilder::new(max_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("max_a")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_expr, max_expr],- expected_before: Some("DynamicFilter [ empty ]"),- expected_after: Some("DynamicFilter [ a@0 < 1 OR a@0 > 8 ]"),- scan_support: true,- })- .await;-}--/// `MIN(a), MAX(b)`: Pushdown dynamic filter like `(a<1) or (b>9)`-#[tokio::test]-async fn test_aggregate_dynamic_filter_min_max_different_columns() {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Int32, true),- ]));- let batches = vec![- record_batch!(("a", Int32, [5, 1, 3, 8]), ("b", Int32, [7, 2, 4, 9])).unwrap(),- ];-- let min_expr =- AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .unwrap();- let max_expr =- AggregateExprBuilder::new(max_udaf(), vec![col("b", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("max_b")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_expr, max_expr],- expected_before: Some("DynamicFilter [ empty ]"),- expected_after: Some("DynamicFilter [ a@0 < 1 OR b@1 > 9 ]"),- scan_support: true,- })- .await;-}--/// Mix of supported/unsupported aggregates retains only the valid ones.-/// `MIN(a), MAX(a), MAX(b), MIN(c+1)`: Pushdown dynamic filter like `(a<1) or (a>8) OR (b>12)`-#[tokio::test]-async fn test_aggregate_dynamic_filter_multiple_mixed_expressions() {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, true),- Field::new("b", DataType::Int32, true),- Field::new("c", DataType::Int32, true),- ]));- let batches = vec![- record_batch!(- ("a", Int32, [5, 1, 3, 8]),- ("b", Int32, [10, 4, 6, 12]),- ("c", Int32, [100, 70, 90, 110])- )- .unwrap(),- ];-- let min_a = AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .unwrap();- let max_a = AggregateExprBuilder::new(max_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("max_a")- .build()- .unwrap();- let max_b = AggregateExprBuilder::new(max_udaf(), vec![col("b", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("max_b")- .build()- .unwrap();- let expr_c: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- col("c", &schema).unwrap(),- Operator::Plus,- Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),- ));- let min_c_expr = AggregateExprBuilder::new(min_udaf(), vec![expr_c])- .schema(Arc::clone(&schema))- .alias("min_c_plus_one")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_a, max_a, max_b, min_c_expr],- expected_before: Some("DynamicFilter [ empty ]"),- expected_after: Some("DynamicFilter [ a@0 < 1 OR a@0 > 8 OR b@1 > 12 ]"),- scan_support: true,- })- .await;-}--/// Don't tighten the dynamic filter if all inputs are null-#[tokio::test]-async fn test_aggregate_dynamic_filter_min_all_nulls() {- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));- let batches = vec![record_batch!(("a", Int32, [None, None, None, None])).unwrap()];-- let min_expr =- AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema).unwrap()])- .schema(Arc::clone(&schema))- .alias("min_a")- .build()- .unwrap();-- run_aggregate_dyn_filter_case(AggregateDynFilterCase {- schema,- batches,- aggr_exprs: vec![min_expr],- expected_before: Some("DynamicFilter [ empty ]"),- // After reading the input it hasn't a meaningful bound to update, so the- // predicate `true` means don't filter out anything- expected_after: Some("DynamicFilter [ true ]"),- scan_support: true,- })- .await;-}--/// Test aggregate dynamic filter is working when reading parquet files-///-/// Runs 'select max(id) from test_table where id > 1', and ensure some file ranges-/// pruned by the dynamic filter.-#[tokio::test]-async fn test_aggregate_dynamic_filter_parquet_e2e() {- let config = SessionConfig::new()- .with_collect_statistics(true)- .with_target_partitions(2)- .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true)- .set_bool("datafusion.execution.parquet.pushdown_filters", true);- let ctx = SessionContext::new_with_config(config);-- let data_path = format!(- "{}/tests/data/test_statistics_per_partition/",- env!("CARGO_MANIFEST_DIR")- );-- ctx.register_parquet("test_table", &data_path, ParquetReadOptions::default())- .await- .unwrap();-- // partition 1:- // files: ..03-01(id=4), ..03-02(id=3)- // partition 1:- // files: ..03-03(id=2), ..03-04(id=1)- //- // In partition 1, after reading the first file, the dynamic filter will be update- // to "id > 4", so the `..03-02` file must be able to get pruned out- let df = ctx- .sql("explain analyze select max(id) from test_table where id > 1")- .await- .unwrap();-- let result = df.collect().await.unwrap();-- let formatted = pretty_format_batches(&result).unwrap();- let explain_analyze = format!("{formatted}");-- // Capture "2" from "files_ranges_pruned_statistics=4 total → 2 matched"- let re = Regex::new(- r"files_ranges_pruned_statistics\s*=\s*(\d+)\s*total\s*[→>\-]\s*(\d+)\s*matched",- )- .unwrap();-- if let Some(caps) = re.captures(&explain_analyze) {- let matched_num: i32 = caps[2].parse().unwrap();- assert!(- matched_num < 4,- "Total 4 files, if some pruned, the matched count is < 4"- );- } else {- unreachable!("metrics should exist")- }-}+//+// The end-to-end min/max dynamic filter cases (simple/min/max/mixed/all-nulls)+// have been ported to+// `datafusion/sqllogictest/test_files/push_down_filter_regression.slt`.+// The `run_aggregate_dyn_filter_case` harness used to drive them was removed+// along with the test functions./// Non-partial (Single) aggregates should skip dynamic filter initialization.#[test]@@ -3115,139 +1806,6 @@ fn test_aggregate_dynamic_filter_not_created_for_single_mode() {);}-#[tokio::test]-async fn test_aggregate_filter_pushdown() {- // Test that filters can pass through AggregateExec even with aggregate functions- // when the filter references grouping columns- // Simulates: SELECT a, COUNT(b) FROM table WHERE a = 'x' GROUP BY a-- let batches = vec![- record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),- ];-- let scan = TestScanBuilder::new(schema())- .with_support(true)- .with_batches(batches)- .build();-- // Create an aggregate: GROUP BY a with COUNT(b)- let group_by = PhysicalGroupBy::new_single(vec![(- col("a", &schema()).unwrap(),- "a".to_string(),- )]);-- // Add COUNT aggregate- let count_expr =- AggregateExprBuilder::new(count_udaf(), vec![col("b", &schema()).unwrap()])- .schema(schema())- .alias("count")- .build()- .unwrap();-- let aggregate = Arc::new(- AggregateExec::try_new(- AggregateMode::Partial,- group_by,- vec![count_expr.into()], // Has aggregate function- vec![None], // No filter on the aggregate function- Arc::clone(&scan),- schema(),- )- .unwrap(),- );-- // Add a filter on the grouping column 'a'- let predicate = col_lit_predicate("a", "x", &schema());- let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap())- as Arc<dyn ExecutionPlan>;-- // Even with aggregate functions, filter on grouping column should be pushed through- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),- @r"- OptimizationTest:- input:- - FilterExec: a@0 = x- - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count], ordering_mode=Sorted- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = x- "- );-}--#[tokio::test]-async fn test_no_pushdown_filter_on_aggregate_result() {- // Test that filters on aggregate results (not grouping columns) are NOT pushed through- // SELECT a, COUNT(b) as cnt FROM table GROUP BY a HAVING cnt > 5- // The filter on 'cnt' cannot be pushed down because it's an aggregate result-- let batches = vec![- record_batch!(("a", Utf8, ["x", "y"]), ("b", Utf8, ["foo", "bar"])).unwrap(),- ];-- let scan = TestScanBuilder::new(schema())- .with_support(true)- .with_batches(batches)- .build();-- // Create an aggregate: GROUP BY a with COUNT(b)- let group_by = PhysicalGroupBy::new_single(vec![(- col("a", &schema()).unwrap(),- "a".to_string(),- )]);-- // Add COUNT aggregate- let count_expr =- AggregateExprBuilder::new(count_udaf(), vec![col("b", &schema()).unwrap()])- .schema(schema())- .alias("count")- .build()- .unwrap();-- let aggregate = Arc::new(- AggregateExec::try_new(- AggregateMode::Partial,- group_by,- vec![count_expr.into()],- vec![None],- Arc::clone(&scan),- schema(),- )- .unwrap(),- );-- // Add a filter on the aggregate output column- // This simulates filtering on COUNT result, which should NOT be pushed through- let agg_schema = aggregate.schema();- let predicate = Arc::new(BinaryExpr::new(- Arc::new(Column::new_with_schema("count[count]", &agg_schema).unwrap()),- Operator::Gt,- Arc::new(Literal::new(ScalarValue::Int64(Some(5)))),- ));- let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap())- as Arc<dyn ExecutionPlan>;-- // The filter should NOT be pushed through the aggregate since it's on an aggregate result- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new(), true),- @r"- OptimizationTest:- input:- - FilterExec: count[count]@1 > 5- - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - FilterExec: count[count]@1 > 5- - AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- "- );-}-#[test]fn test_pushdown_filter_on_non_first_grouping_column() {// Test that filters on non-first grouping columns are still pushed down@@ -3628,209 +2186,61 @@ fn test_pushdown_through_aggregate_grouping_sets_with_reordered_input() {aggregate_expr,vec![None],projection,- reordered_schema,- )- .unwrap(),- );-- let agg_output_schema = aggregate.schema();-- // Filter on b (present in all grouping sets) should be pushed down- let predicate = col_lit_predicate("b", "bar", &agg_output_schema);- let plan = Arc::new(FilterExec::try_new(predicate, aggregate.clone()).unwrap());-- insta::assert_snapshot!(- OptimizationTest::new(plan, FilterPushdown::new(), true),- @r"- OptimizationTest:- input:- - FilterExec: b@1 = bar- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]- - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])- - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar- "- );-- // Filter on a (missing from second grouping set) should not be pushed down- let predicate = col_lit_predicate("a", "foo", &agg_output_schema);- let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());-- insta::assert_snapshot!(- OptimizationTest::new(plan, FilterPushdown::new(), true),- @r"- OptimizationTest:- input:- - FilterExec: a@0 = foo- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]- - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- output:- Ok:- - FilterExec: a@0 = foo- - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]- - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- "- );-}--/// Regression test for https://github.com/apache/datafusion/issues/21065.-///-/// Given a plan similar to the following, ensure that the filter is pushed down-/// through an AggregateExec whose input columns are reordered by a ProjectionExec.-#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_through_aggregate_with_reordered_input() {- // Build side- let build_batches = vec![record_batch!(("a", Utf8, ["h1", "h2"])).unwrap()];- let build_schema =- Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false)]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Probe side- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["h1", "h2", "h3", "h4"]),- ("value", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap(),- ];- let probe_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("value", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // ProjectionExec reorders (a, value) → (value, a)- let reordered_schema = Arc::new(Schema::new(vec![- Field::new("value", DataType::Float64, false),- Field::new("a", DataType::Utf8, false),- ]));- let projection = Arc::new(- ProjectionExec::try_new(- vec![- (col("value", &probe_schema).unwrap(), "value".to_string()),- (col("a", &probe_schema).unwrap(), "a".to_string()),- ],- probe_scan,- )- .unwrap(),- );-- // AggregateExec: GROUP BY a@1, min(value@0)- let aggregate_expr = vec![- AggregateExprBuilder::new(- min_udaf(),- vec![col("value", &reordered_schema).unwrap()],- )- .schema(reordered_schema.clone())- .alias("min_value")- .build()- .map(Arc::new)- .unwrap(),- ];- let group_by = PhysicalGroupBy::new_single(vec![(- col("a", &reordered_schema).unwrap(), // a@1 in input- "a".to_string(),- )]);-- let aggregate = Arc::new(- AggregateExec::try_new(- AggregateMode::Single,- group_by,- aggregate_expr,- vec![None],- projection,- reordered_schema,- )- .unwrap(),- );-- // Aggregate output schema: (a@0, min_value@1)- let agg_output_schema = aggregate.schema();-- // Join the build and probe side- let plan = Arc::new(- HashJoinExec::try_new(- build_scan,- aggregate,- vec![(- col("a", &build_schema).unwrap(),- col("a", &agg_output_schema).unwrap(),- )],- None,- &JoinType::Inner,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,+ reordered_schema,).unwrap(),- ) as Arc<dyn ExecutionPlan>;+ );++ let agg_output_schema = aggregate.schema();++ // Filter on b (present in all grouping sets) should be pushed down+ let predicate = col_lit_predicate("b", "bar", &agg_output_schema);+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate.clone()).unwrap());- // The HashJoin's dynamic filter on `a` should push- // through the aggregate and reach the probe-side DataSource.insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),+ OptimizationTest::new(plan, FilterPushdown::new(), true),@r"OptimizationTest:input:- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true- - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]- - ProjectionExec: expr=[value@1 as value, a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true+ - FilterExec: b@1 = bar+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=trueoutput:Ok:- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true- - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]- - ProjectionExec: expr=[value@1 as value, a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt], ordering_mode=PartiallySorted([1])+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar");- // Actually execute the plan to verify the dynamic filter is populated- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();-- let session_config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(session_config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let mut stream = plan.execute(0, Arc::clone(&task_ctx)).unwrap();- stream.next().await.unwrap().unwrap();+ // Filter on a (missing from second grouping set) should not be pushed down+ let predicate = col_lit_predicate("a", "foo", &agg_output_schema);+ let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap());- // After execution, the dynamic filter should be populated with valuesinsta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),+ OptimizationTest::new(plan, FilterPushdown::new(), true),@r"- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a], file_type=test, pushdown_supported=true- - AggregateExec: mode=Single, gby=[a@1 as a], aggr=[min_value]- - ProjectionExec: expr=[value@1 as value, a@0 as a]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, value], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ]+ OptimizationTest:+ input:+ - FilterExec: a@0 = foo+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ output:+ Ok:+ - FilterExec: a@0 = foo+ - AggregateExec: mode=Final, gby=[(a@1 as a, b@2 as b), (NULL as a, b@2 as b)], aggr=[cnt]+ - ProjectionExec: expr=[c@2 as c, a@0 as a, b@1 as b]+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true");}+/// Regression test for https://github.com/apache/datafusion/issues/21065.+///+/// Given a plan similar to the following, ensure that the filter is pushed down+/// through an AggregateExec whose input columns are reordered by a ProjectionExec.#[test]fn test_pushdown_with_computed_grouping_key() {// Test filter pushdown with computed grouping expression@@ -3892,6 +2302,11 @@ fn test_pushdown_with_computed_grouping_key() {);}+// Not portable to sqllogictest: in CollectLeft (the mode SQL picks for small+// data), an empty build side short-circuits the HashJoin and the probe scan+// is never executed, so its dynamic filter stays at `[ empty ]` rather than+// collapsing to `[ false ]`. The Rust test uses PartitionMode::Partitioned+// on a hand-wired plan, which does trigger the `false` path.#[tokio::test]async fn test_hashjoin_dynamic_filter_all_partitions_empty() {use datafusion_common::JoinType;@@ -4024,145 +2439,9 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() {);}-#[tokio::test]-async fn test_hashjoin_dynamic_filter_with_nulls() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Test scenario where build side has NULL values in join keys- // This validates NULL handling in bounds computation and filter generation-- // Create build side with NULL values- let build_batch = RecordBatch::try_new(- Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, true), // nullable- Field::new("b", DataType::Int32, true), // nullable- ])),- vec![- Arc::new(StringArray::from(vec![Some("aa"), None, Some("ab")])),- Arc::new(Int32Array::from(vec![Some(1), Some(2), None])),- ],- )- .unwrap();- let build_batches = vec![build_batch];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, true),- Field::new("b", DataType::Int32, true),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with nullable fields- let probe_batch = RecordBatch::try_new(- Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, true),- Field::new("b", DataType::Int32, true),- Field::new("c", DataType::Float64, false),- ])),- vec![- Arc::new(StringArray::from(vec![- Some("aa"),- Some("ab"),- Some("ac"),- None,- ])),- Arc::new(Int32Array::from(vec![Some(1), Some(3), Some(4), Some(5)])),- Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])),- ],- )- .unwrap();- let probe_batches = vec![probe_batch];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, true),- Field::new("b", DataType::Int32, true),- Field::new("c", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec in CollectLeft mode (simpler for this test)- let on = vec![- (- col("a", &build_side_schema).unwrap(),- col("a", &probe_side_schema).unwrap(),- ),- (- col("b", &build_side_schema).unwrap(),- col("b", &probe_side_schema).unwrap(),- ),- ];- let plan = Arc::new(- HashJoinExec::try_new(- build_scan,- Arc::clone(&probe_scan),- on,- None,- &JoinType::Inner,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- );-- // Apply the filter pushdown optimizer- let mut config = SessionConfig::new();- config.options_mut().execution.parquet.pushdown_filters = true;- let optimizer = FilterPushdown::new_post_optimization();- let plan = optimizer.optimize(plan, config.options()).unwrap();-- insta::assert_snapshot!(- format_plan_for_test(&plan),- @r"- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- "- );-- // Put some data through the plan to check that the filter is updated to reflect the TopK state- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- // Execute all partitions (required for partitioned hash join coordination)- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();-- // Test that filters are pushed down correctly to each side of the join- insta::assert_snapshot!(- format_plan_for_test(&plan),- @r"- - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= 1 AND b@1 <= 2 AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:1}, {c0:,c1:2}, {c0:ab,c1:}]) ]- "- );-- #[rustfmt::skip]- let expected = [- "+----+---+----+---+-----+",- "| a | b | a | b | c |",- "+----+---+----+---+-----+",- "| aa | 1 | aa | 1 | 1.0 |",- "+----+---+----+---+-----+",- ];- assert_batches_eq!(&expected, &batches);-}--/// Test that when hash_join_inlist_pushdown_max_size is set to a very small value,-/// the HashTable strategy is used instead of InList strategy, even with small build sides.-/// This test is identical to test_hashjoin_dynamic_filter_pushdown_partitioned except-/// for the config setting that forces the HashTable strategy.+// Not portable to sqllogictest: same reason as+// test_hashjoin_dynamic_filter_pushdown_partitioned — hand-wires+// PartitionMode::Partitioned, which SQL never picks for small parquet inputs.#[tokio::test]async fn test_hashjoin_hash_table_pushdown_partitioned() {use datafusion_common::JoinType;@@ -4325,10 +2604,10 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() {);}-/// Test that when hash_join_inlist_pushdown_max_size is set to a very small value,-/// the HashTable strategy is used instead of InList strategy in CollectLeft mode.-/// This test is identical to test_hashjoin_dynamic_filter_pushdown_collect_left except-/// for the config setting that forces the HashTable strategy.+// Ported to push_down_filter_parquet.slt (`hl_build`/`hl_probe` fixture).+// Rust version retained only because the slt port cannot hand-wire the+// RepartitionExec-above-probe shape this test uses; the hash_lookup vs+// IN (SET) invariant is captured in the slt port.#[tokio::test]async fn test_hashjoin_hash_table_pushdown_collect_left() {use datafusion_common::JoinType;@@ -4397,145 +2676,10 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {col("b", &probe_side_schema).unwrap(),),];- let hash_join = Arc::new(- HashJoinExec::try_new(- build_scan,- probe_repartition,- on,- None,- &JoinType::Inner,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- );-- // Top-level CoalescePartitionsExec- let cp = Arc::new(CoalescePartitionsExec::new(hash_join)) as Arc<dyn ExecutionPlan>;- // Add a sort for deterministic output- let plan = Arc::new(SortExec::new(- LexOrdering::new(vec![PhysicalSortExpr::new(- col("a", &probe_side_schema).unwrap(),- SortOptions::new(true, false), // descending, nulls_first- )])- .unwrap(),- cp,- )) as Arc<dyn ExecutionPlan>;-- // Apply the optimization with config setting that forces HashTable strategy- let session_config = SessionConfig::default()- .with_batch_size(10)- .set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1)- .set_bool("datafusion.execution.parquet.pushdown_filters", true)- .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true);- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, session_config.options())- .unwrap();- let session_ctx = SessionContext::new_with_config(session_config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();-- // Verify that hash_lookup is used instead of IN (SET)- let plan_str = format_plan_for_test(&plan).to_string();- assert!(- plan_str.contains("hash_lookup"),- "Expected hash_lookup in plan but got: {plan_str}"- );- assert!(- !plan_str.contains("IN (SET)"),- "Expected no IN (SET) in plan but got: {plan_str}"- );-- let result = format!("{}", pretty_format_batches(&batches).unwrap());-- let probe_scan_metrics = probe_scan.metrics().unwrap();-- // The probe side had 4 rows, but after applying the dynamic filter only 2 rows should remain.- assert_eq!(probe_scan_metrics.output_rows().unwrap(), 2);-- // Results should be identical to the InList version- insta::assert_snapshot!(- result,- @r"- +----+----+-----+----+----+-----+- | a | b | c | a | b | e |- +----+----+-----+----+----+-----+- | ab | bb | 2.0 | ab | bb | 2.0 |- | aa | ba | 1.0 | aa | ba | 1.0 |- +----+----+-----+----+----+-----+- ",- );-}--/// Test HashTable strategy with integer multi-column join keys.-/// Verifies that hash_lookup works correctly with integer data types.-#[tokio::test]-async fn test_hashjoin_hash_table_pushdown_integer_keys() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with integer keys- let build_batches = vec![- record_batch!(- ("id1", Int32, [1, 2]),- ("id2", Int32, [10, 20]),- ("value", Float64, [100.0, 200.0])- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("id1", DataType::Int32, false),- Field::new("id2", DataType::Int32, false),- Field::new("value", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more integer rows- let probe_batches = vec![- record_batch!(- ("id1", Int32, [1, 2, 3, 4]),- ("id2", Int32, [10, 20, 30, 40]),- ("data", Utf8, ["a", "b", "c", "d"])- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("id1", DataType::Int32, false),- Field::new("id2", DataType::Int32, false),- Field::new("data", DataType::Utf8, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create join on multiple integer columns- let on = vec![- (- col("id1", &build_side_schema).unwrap(),- col("id1", &probe_side_schema).unwrap(),- ),- (- col("id2", &build_side_schema).unwrap(),- col("id2", &probe_side_schema).unwrap(),- ),- ];- let plan = Arc::new(+ let hash_join = Arc::new(HashJoinExec::try_new(build_scan,- Arc::clone(&probe_scan),+ probe_repartition,on,None,&JoinType::Inner,@@ -4547,7 +2691,19 @@ async fn test_hashjoin_hash_table_pushdown_integer_keys() {.unwrap(),);- // Apply optimization with forced HashTable strategy+ // Top-level CoalescePartitionsExec+ let cp = Arc::new(CoalescePartitionsExec::new(hash_join)) as Arc<dyn ExecutionPlan>;+ // Add a sort for deterministic output+ let plan = Arc::new(SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr::new(+ col("a", &probe_side_schema).unwrap(),+ SortOptions::new(true, false), // descending, nulls_first+ )])+ .unwrap(),+ cp,+ )) as Arc<dyn ExecutionPlan>;++ // Apply the optimization with config setting that forces HashTable strategylet session_config = SessionConfig::default().with_batch_size(10).set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1)@@ -4567,7 +2723,7 @@ async fn test_hashjoin_hash_table_pushdown_integer_keys() {.await.unwrap();- // Verify hash_lookup is used+ // Verify that hash_lookup is used instead of IN (SET)let plan_str = format_plan_for_test(&plan).to_string();assert!(plan_str.contains("hash_lookup"),@@ -4581,22 +2737,30 @@ async fn test_hashjoin_hash_table_pushdown_integer_keys() {let result = format!("{}", pretty_format_batches(&batches).unwrap());let probe_scan_metrics = probe_scan.metrics().unwrap();- // Only 2 rows from probe side match the build side++ // The probe side had 4 rows, but after applying the dynamic filter only 2 rows should remain.assert_eq!(probe_scan_metrics.output_rows().unwrap(), 2);+ // Results should be identical to the InList versioninsta::assert_snapshot!(result,@r"- +-----+-----+-------+-----+-----+------+- | id1 | id2 | value | id1 | id2 | data |- +-----+-----+-------+-----+-----+------+- | 1 | 10 | 100.0 | 1 | 10 | a |- | 2 | 20 | 200.0 | 2 | 20 | b |- +-----+-----+-------+-----+-----+------++ +----+----+-----+----+----+-----++ | a | b | c | a | b | e |+ +----+----+-----+----+----+-----++ | ab | bb | 2.0 | ab | bb | 2.0 |+ | aa | ba | 1.0 | aa | ba | 1.0 |+ +----+----+-----+----+----+-----+",);}+// Not portable to sqllogictest: asserts on `HashJoinExec::dynamic_filter_for_test().is_used()`+// which is a debug-only API. The observable behavior (probe-side scan+// receiving the dynamic filter when the data source supports it) is+// already covered by the simpler CollectLeft port in push_down_filter_parquet.slt;+// the with_support(false) branch has no SQL analog (parquet always supports+// pushdown).#[tokio::test]async fn test_hashjoin_dynamic_filter_pushdown_is_used() {use datafusion_common::JoinType;@@ -4685,7 +2849,13 @@ async fn test_hashjoin_dynamic_filter_pushdown_is_used() {}}-/// Regression test for https://github.com/apache/datafusion/issues/20109+/// Regression test for https://github.com/apache/datafusion/issues/20109.+///+/// Not portable to sqllogictest: the regression specifically targets the+/// physical FilterPushdown rule running over *stacked* FilterExecs with+/// projections on a MemorySourceConfig. In SQL the logical optimizer+/// collapses the two filters before physical planning, so the stacked+/// FilterExec shape this test exercises is unreachable.#[tokio::test]async fn test_filter_with_projection_pushdown() {use arrow::array::{Int64Array, RecordBatch, StringArray};@@ -4772,7 +2942,11 @@ async fn test_filter_with_projection_pushdown() {assert_batches_eq!(expected, &result);}-/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree+/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree.+///+/// Not portable to sqllogictest: asserts by walking the plan tree with+/// `apply_expressions` + `downcast_ref::<DynamicFilterPhysicalExpr>` and+/// counting nodes. Neither API is observable from SQL.#[tokio::test]async fn test_discover_dynamic_filters_via_expressions_api() {use datafusion_common::JoinType;@@ -4872,277 +3046,3 @@ async fn test_discover_dynamic_filters_via_expressions_api() {"After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}");}--#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_left_join() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values (some won't match)- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec with Left join and CollectLeft mode- let on = vec![- (- col("a", &build_side_schema).unwrap(),- col("a", &probe_side_schema).unwrap(),- ),- (- col("b", &build_side_schema).unwrap(),- col("b", &probe_side_schema).unwrap(),- ),- ];- let plan = Arc::new(- HashJoinExec::try_new(- build_scan,- Arc::clone(&probe_scan),- on,- None,- &JoinType::Left,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- ) as Arc<dyn ExecutionPlan>;-- // Expect the dynamic filter predicate to be pushed down into the probe side DataSource- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true- output:- Ok:- - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- ",- );-- // Actually apply the optimization and execute the plan- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();-- // Test that dynamic filter linking survives with_new_children- let children = plan.children().into_iter().map(Arc::clone).collect();- let plan = plan.with_new_children(children).unwrap();-- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();-- // After execution, verify the dynamic filter was populated with bounds and IN-list- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ]- "- );-- // Verify result correctness: left join preserves all build (left) rows.- // All build rows match probe rows here, so we get 2 matched rows.- // The dynamic filter pruned unmatched probe rows (ac, ad) at scan time,- // which is safe because those probe rows would never match any build row.- let result = format!("{}", pretty_format_batches(&batches).unwrap());- insta::assert_snapshot!(- result,- @r"- +----+----+-----+----+----+-----+- | a | b | c | a | b | e |- +----+----+-----+----+----+-----+- | aa | ba | 1.0 | aa | ba | 1.0 |- | ab | bb | 2.0 | ab | bb | 2.0 |- +----+----+-----+----+----+-----+- "- );-}--#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_left_semi_join() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0])- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values (some won't match)- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0])- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();-- // Create HashJoinExec with LeftSemi join and CollectLeft mode- let on = vec![- (- col("a", &build_side_schema).unwrap(),- col("a", &probe_side_schema).unwrap(),- ),- (- col("b", &build_side_schema).unwrap(),- col("b", &probe_side_schema).unwrap(),- ),- ];- let plan = Arc::new(- HashJoinExec::try_new(- build_scan,- Arc::clone(&probe_scan),- on,- None,- &JoinType::LeftSemi,- None,- PartitionMode::CollectLeft,- datafusion_common::NullEquality::NullEqualsNothing,- false,- )- .unwrap(),- ) as Arc<dyn ExecutionPlan>;-- // Expect the dynamic filter predicate to be pushed down into the probe side DataSource- insta::assert_snapshot!(- OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),- @r"- OptimizationTest:- input:- - HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true- output:- Ok:- - HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]- ",- );-- // Actually apply the optimization and execute the plan- let mut config = ConfigOptions::default();- config.execution.parquet.pushdown_filters = true;- config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();-- // Test that dynamic filter linking survives with_new_children- let children = plan.children().into_iter().map(Arc::clone).collect();- let plan = plan.with_new_children(children).unwrap();-- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();-- // After execution, verify the dynamic filter was populated with bounds and IN-list- insta::assert_snapshot!(- format!("{}", format_plan_for_test(&plan)),- @r"- - HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)]- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true- - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ]- "- );-- // Verify result correctness: left semi join returns only build (left) rows- // that have at least one matching probe row. Output schema is build-side columns only.- let result = format!("{}", pretty_format_batches(&batches).unwrap());- insta::assert_snapshot!(- result,- @r"- +----+----+-----+- | a | b | c |- +----+----+-----+- | aa | ba | 1.0 |- | ab | bb | 2.0 |- +----+----+-----+- "- );-}diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex ab6847e1c4..8469c32a17 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -166,6 +166,863 @@ drop table small_table;statement okdrop table large_table;++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_topk_dynamic_filter_pushdown_integration+#+# Integration test for dynamic filter pushdown with TopK:+# a TopK over an ordered parquet file emits a dynamic filter that should+# get pushed into the parquet scan and prune every row group except the+# first (128 rows), visible via `pushdown_rows_matched` / `pushdown_rows_pruned`.+#+# `t * t` (rather than `t`) is used as the sort expression so the+# order-pushdown optimizer does not rewrite the TopK away.+########++statement ok+set datafusion.execution.parquet.max_row_group_size = 128;++query I+COPY (+ SELECT 1372708800 + value AS t+ FROM generate_series(0, 99999)+ ORDER BY t+) TO 'test_files/scratch/push_down_filter_parquet/topk_pushdown.parquet'+STORED AS PARQUET;+----+100000++statement ok+CREATE EXTERNAL TABLE topk_pushdown+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_pushdown.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE SELECT t FROM topk_pushdown ORDER BY t * t LIMIT 10;+----+Plan with Metrics+01)SortExec: TopK(fetch=10), expr=[t@0 * t@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[t@0 * t@0 < 1884329474306198481], metrics=[output_rows=10, output_batches=1, row_replacements=10]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_pushdown.parquet]]}, projection=[t], output_ordering=[t@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ t@0 * t@0 < 1884329474306198481 ], metrics=[output_rows=128, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=782 total → 782 matched, row_groups_pruned_bloom_filter=782 total → 782 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=128, pushdown_rows_pruned=99.87 K, predicate_cache_inner_records=128, predicate_cache_records=128, scan_efficiency_ratio=64.87% (258.7 K/398.8 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+reset datafusion.execution.parquet.max_row_group_size;++statement ok+drop table topk_pushdown;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_topk_dynamic_filter_pushdown+#+# A `ORDER BY b DESC LIMIT 1` over a parquet file should emit a dynamic+# filter on the TopK that gets pushed into the scan's predicate, visible+# in EXPLAIN ANALYZE as `DynamicFilter [ b > <max> ]` tightened to the+# running max seen by the TopK during execution.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'bd', 1.0),+ ('ab', 'bc', 2.0),+ ('ac', 'bb', 2.0),+ ('ad', 'ba', 1.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/topk_single_col.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE topk_single_col (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_single_col.parquet';++# Data-correctness check — top row by b desc is (aa, bd, 1).+query TTR+SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;+----+aa bd 1++# Before execution: a dynamic filter is emitted but still empty+# (DynamicFilter [ empty ]) on the parquet scan.+query TT+EXPLAIN SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;+----+physical_plan+01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ empty ]++statement ok+set datafusion.explain.analyze_categories = 'rows';++# After execution: the dynamic filter is tightened to `b > <max_seen>`.+query TT+EXPLAIN ANALYZE SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;+----+Plan with Metrics+01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false], filter=[b@1 IS NULL OR b@1 > bd], metrics=[output_rows=1, output_batches=1, row_replacements=1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=22.37% (240/1.07 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table topk_single_col;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_topk_dynamic_filter_pushdown_multi_column_sort+#+# TopK with a compound sort key (b ASC NULLS LAST, a DESC). The dynamic+# filter emitted to the scan should encode the tiebreaker as a compound+# predicate of the form `b < <k1> OR (b = <k1> AND a > <k2>)`.+# With fetch=2 the top-2 rows are (ad, ba) and (ac, bb), so the final+# filter reads `b < bb OR (b = bb AND a > ac)`.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('ac', 'bb', 2.0),+ ('ad', 'ba', 1.0),+ ('aa', 'bc', 1.0),+ ('ab', 'bd', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE topk_multi_col (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet';++# Data-correctness: confirm the top-2 rows come out in the expected order.+query TTR+SELECT * FROM topk_multi_col ORDER BY b ASC NULLS LAST, a DESC LIMIT 2;+----+ad ba 1+ac bb 2++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE SELECT * FROM topk_multi_col ORDER BY b ASC NULLS LAST, a DESC LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false], filter=[b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac)], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=22.37% (240/1.07 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table topk_multi_col;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_dynamic_filter_pushdown+#+# CollectLeft hash join on two equi-join keys: the dynamic filter emitted+# by the build side should cover both keys and include an `IN (SET)`+# predicate over `struct(a, b)` for the rows it collected.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/join_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0),+ ('ac', 'bc', 3.0),+ ('ad', 'bd', 4.0)+ ) AS v(a, b, e)+) TO 'test_files/scratch/push_down_filter_parquet/join_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE join_build (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/join_build.parquet';++statement ok+CREATE EXTERNAL TABLE join_probe (a VARCHAR, b VARCHAR, e DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/join_probe.parquet';++# Data-correctness: only the 2 build rows should match.+query TTRR+SELECT p.a, p.b, build.c, p.e+FROM join_probe p INNER JOIN join_build AS build+ ON p.a = build.a AND p.b = build.b+ORDER BY p.a;+----+aa ba 1 1+ab bb 2 2++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE+SELECT p.a, p.b, build.c, p.e+FROM join_probe p INNER JOIN join_build AS build+ ON p.a = build.a AND p.b = build.b;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=20.48% (214/1.04 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.78% (246/1.08 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table join_build;++statement ok+drop table join_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_nested_hashjoin_dynamic_filter_pushdown+#+# Nested hash joins: `t1 JOIN (t2 JOIN t3 ON t2.c = t3.d) ON t1.a = t2.b`+# should push dynamic filters down to BOTH the t2 and t3 scans+# (t2 gets a filter on `b`, t3 gets a filter on `d`).+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 1.0),+ ('ab', 2.0)+ ) AS v(a, x)+) TO 'test_files/scratch/push_down_filter_parquet/nested_t1.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ca', 1.0),+ ('ab', 'cb', 2.0),+ ('ac', 'cc', 3.0),+ ('ad', 'cd', 4.0),+ ('ae', 'ce', 5.0)+ ) AS v(b, c, y)+) TO 'test_files/scratch/push_down_filter_parquet/nested_t2.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('ca', 1.0),+ ('cb', 2.0),+ ('cc', 3.0),+ ('cd', 4.0),+ ('ce', 5.0),+ ('cf', 6.0),+ ('cg', 7.0),+ ('ch', 8.0)+ ) AS v(d, z)+) TO 'test_files/scratch/push_down_filter_parquet/nested_t3.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE nested_t1 (a VARCHAR, x DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/nested_t1.parquet';++statement ok+CREATE EXTERNAL TABLE nested_t2 (b VARCHAR, c VARCHAR, y DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/nested_t2.parquet';++statement ok+CREATE EXTERNAL TABLE nested_t3 (d VARCHAR, z DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/nested_t3.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE+SELECT *+FROM nested_t1+INNER JOIN nested_t2 ON nested_t1.a = nested_t2.b+INNER JOIN nested_t3 ON nested_t2.c = nested_t3.d;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c@3, d@0)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, b@0)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t1.parquet]]}, projection=[a, x], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.23% (144/790)]+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t2.parquet]]}, projection=[b, c, y], file_type=parquet, predicate=DynamicFilter [ b@0 >= aa AND b@0 <= ab AND b@0 IN (SET) ([aa, ab]) ], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= aa AND b_null_count@1 != row_count@2 AND b_min@3 <= ab AND (b_null_count@1 != row_count@2 AND b_min@3 <= aa AND aa <= b_max@0 OR b_null_count@1 != row_count@2 AND b_min@3 <= ab AND ab <= b_max@0), required_guarantees=[b in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=5 total → 5 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=3, predicate_cache_inner_records=5, predicate_cache_records=2, scan_efficiency_ratio=23.2% (252/1.09 K)]+05)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t3.parquet]]}, projection=[d, z], file_type=parquet, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND hash_lookup ], pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= ca AND d_null_count@1 != row_count@2 AND d_min@3 <= cb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=8 total → 8 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=6, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=22.12% (184/832)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table nested_t1;++statement ok+drop table nested_t2;++statement ok+drop table nested_t3;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_parent_filter_pushdown+#+# A FilterExec above a HashJoin should split: the single-side predicates+# should be pushed down to each scan, while any cross-side predicate+# stays above the join.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/parent_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0),+ ('ac', 'bc', 3.0),+ ('ad', 'bd', 4.0)+ ) AS v(d, e, f)+) TO 'test_files/scratch/push_down_filter_parquet/parent_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE parent_build (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/parent_build.parquet';++statement ok+CREATE EXTERNAL TABLE parent_probe (d VARCHAR, e VARCHAR, f DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/parent_probe.parquet';++query TT+EXPLAIN+SELECT *+FROM parent_build+INNER JOIN parent_probe ON parent_build.a = parent_probe.d+WHERE parent_build.a = 'aa' AND parent_probe.e = 'ba';+----+physical_plan+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, d@0)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/parent_build.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=a@0 = aa, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= aa AND aa <= a_max@1, required_guarantees=[a in (aa)]+03)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/parent_probe.parquet]]}, projection=[d, e, f], file_type=parquet, predicate=e@1 = ba AND d@0 = aa AND DynamicFilter [ empty ], pruning_predicate=e_null_count@2 != row_count@3 AND e_min@0 <= ba AND ba <= e_max@1 AND d_null_count@6 != row_count@3 AND d_min@4 <= aa AND aa <= d_max@5, required_guarantees=[d in (aa), e in (ba)]++statement ok+drop table parent_build;++statement ok+drop table parent_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_dynamic_filter_pushdown_through_hash_join_with_topk+#+# Composition: a TopK sitting above an InnerJoin should push its dynamic+# filter THROUGH the hash join and into the probe-side scan, where it+# gets combined (AND) with the hash join's own dynamic filter. The+# probe-side predicate should contain BOTH a `DynamicFilter [ d IN (...) ]`+# (from the join) and a `DynamicFilter [ e < <topk_bound> ]` (from the TopK).+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/topk_join_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0),+ ('ac', 'bc', 3.0),+ ('ad', 'bd', 4.0)+ ) AS v(d, e, f)+) TO 'test_files/scratch/push_down_filter_parquet/topk_join_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE topk_join_build (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_join_build.parquet';++statement ok+CREATE EXTERNAL TABLE topk_join_probe (d VARCHAR, e VARCHAR, f DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_join_probe.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE+SELECT topk_join_probe.e+FROM topk_join_build+INNER JOIN topk_join_probe ON topk_join_build.a = topk_join_probe.d+ORDER BY topk_join_probe.e ASC+LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[e@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[e@0 < bb], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, d@0)], projection=[e@2], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_build.parquet]]}, projection=[a], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=6.7% (70/1.04 K)]+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_probe.parquet]]}, projection=[d, e], file_type=parquet, predicate=DynamicFilter [ d@0 >= aa AND d@0 <= ab AND d@0 IN (SET) ([aa, ab]) ] AND DynamicFilter [ e@1 < bb ], pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= aa AND d_null_count@1 != row_count@2 AND d_min@3 <= ab AND (d_null_count@1 != row_count@2 AND d_min@3 <= aa AND aa <= d_max@0 OR d_null_count@1 != row_count@2 AND d_min@3 <= ab AND ab <= d_max@0) AND e_null_count@5 != row_count@2 AND e_min@4 < bb, required_guarantees=[d in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.37% (166/1.08 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table topk_join_build;++statement ok+drop table topk_join_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_topk_with_projection_transformation_on_dyn_filter+#+# When a ProjectionExec sits between a TopK and the parquet scan, the+# dynamic filter emitted by the TopK (on the projected column) must be+# rewritten back to the original scan-side column before it reaches the+# scan. Covers: simple `SELECT a`, column reorder (`SELECT b, a`),+# expression projection (`SELECT a+1 AS a_plus_1`), and column alias+# shadowing (`SELECT a+1 AS a`).+########++statement ok+COPY (+ SELECT * FROM (VALUES+ (1, 'x', 1.0),+ (2, 'y', 2.0),+ (3, 'z', 3.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/topk_proj.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE topk_proj (a INT, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/topk_proj.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++# Case 1: reorder — `SELECT b, a` — filter on the TopK uses projected column+# `a`, but the scan predicate must reference the original `a@0`.+query TT+EXPLAIN ANALYZE SELECT b, a FROM topk_proj ORDER BY a LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[a@1 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@1 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]++# Case 2: prune — `SELECT a` — filter stays as `a < 2` on the scan.+query TT+EXPLAIN ANALYZE SELECT a FROM topk_proj ORDER BY a LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=7.09% (79/1.11 K)]++# Case 3: expression — `SELECT a+1 AS a_plus_1` — the TopK filter is on+# `a_plus_1`, the scan predicate must read `a@0 + 1`.+query TT+EXPLAIN ANALYZE SELECT a + 1 AS a_plus_1, b FROM topk_proj ORDER BY a_plus_1 LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a_plus_1@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a_plus_1, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]++# Case 4: alias shadowing — `SELECT a+1 AS a` — the projection renames+# `a+1` to `a`, so the TopK's `a < 3` must still be rewritten to+# `a@0 + 1 < 3` on the scan.+query TT+EXPLAIN ANALYZE SELECT a + 1 AS a, b FROM topk_proj ORDER BY a LIMIT 2;+----+Plan with Metrics+01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table topk_proj;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_dynamic_filter_pushdown_through_aggregate_with_reordered_input+#+# The HashJoin's dynamic filter on the join key should propagate down+# through a GROUP BY aggregate and land on the underlying parquet scan.+########++statement ok+COPY (+ SELECT * FROM (VALUES ('h1'), ('h2')) AS v(a)+) TO 'test_files/scratch/push_down_filter_parquet/join_agg_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('h1', 1.0),+ ('h2', 2.0),+ ('h3', 3.0),+ ('h4', 4.0)+ ) AS v(a, value)+) TO 'test_files/scratch/push_down_filter_parquet/join_agg_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE join_agg_build (a VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/join_agg_build.parquet';++statement ok+CREATE EXTERNAL TABLE join_agg_probe (a VARCHAR, value DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/join_agg_probe.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE+SELECT b.a, agg.min_value+FROM join_agg_build b+INNER JOIN (+ SELECT a, MIN(value) AS min_value FROM join_agg_probe GROUP BY a+) agg ON b.a = agg.a;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)], projection=[a@0, min_value@2], metrics=[output_rows=2, output_batches=2, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_agg_build.parquet]]}, projection=[a], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=15.32% (70/457)]+03)--ProjectionExec: expr=[a@0 as a, min(join_agg_probe.value)@1 as min_value], metrics=[output_rows=2, output_batches=2]+04)----AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[min(join_agg_probe.value)], metrics=[output_rows=2, output_batches=2, spill_count=0, spilled_rows=0]+05)------RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1, metrics=[output_rows=2, output_batches=2, spill_count=0, spilled_rows=0]+06)--------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[min(join_agg_probe.value)], metrics=[output_rows=2, output_batches=1, spill_count=0, spilled_rows=0, skipped_aggregation_rows=0, reduction_factor=100% (2/2)]+07)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_agg_probe.parquet]]}, projection=[a, value], file_type=parquet, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= h1 AND a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND (a_null_count@1 != row_count@2 AND a_min@3 <= h1 AND h1 <= a_max@0 OR a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND h2 <= a_max@0), required_guarantees=[a in (h1, h2)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=2, scan_efficiency_ratio=19.81% (163/823)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table join_agg_build;++statement ok+drop table join_agg_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_dynamic_filter_with_nulls+#+# HashJoin build side contains NULL join-key values. The dynamic filter+# emitted to the probe scan should still correctly handle NULLs.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 1),+ (CAST(NULL AS VARCHAR), 2),+ ('ab', CAST(NULL AS INT))+ ) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_parquet/nulls_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 1, 1.0),+ ('ab', 3, 2.0),+ ('ac', 4, 3.0),+ (CAST(NULL AS VARCHAR), 5, 4.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/nulls_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE nulls_build (a VARCHAR, b INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/nulls_build.parquet';++statement ok+CREATE EXTERNAL TABLE nulls_probe (a VARCHAR, b INT, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/nulls_probe.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE+SELECT nulls_build.a, nulls_build.b, nulls_probe.a, nulls_probe.b, nulls_probe.c+FROM nulls_build INNER JOIN nulls_probe+ON nulls_build.a = nulls_probe.a AND nulls_build.b = nulls_probe.b;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=1, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=1, input_rows=1, avg_fanout=100% (1/1), probe_hit_rate=100% (1/1)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nulls_build.parquet]]}, projection=[a, b], file_type=parquet, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.6% (144/774)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nulls_probe.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= 1 AND b@1 <= 2 AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:1}, {c0:,c1:2}, {c0:ab,c1:}]) ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= 1 AND b_null_count@5 != row_count@2 AND b_min@6 <= 2, required_guarantees=[], metrics=[output_rows=1, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=1, pushdown_rows_pruned=3, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=21.1% (237/1.12 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table nulls_build;++statement ok+drop table nulls_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_dynamic_filter_pushdown_left_join+# test_hashjoin_dynamic_filter_pushdown_left_semi_join+#+# For LEFT and LEFT SEMI joins (without a WHERE that would convert them+# to INNER), the HashJoin's dynamic filter should still push into the+# probe scan on the join keys — unmatched probe rows that can never+# contribute to any build match are safely pruned there.+########++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/lj_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0),+ ('ac', 'bc', 3.0),+ ('ad', 'bd', 4.0)+ ) AS v(a, b, e)+) TO 'test_files/scratch/push_down_filter_parquet/lj_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE lj_build (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/lj_build.parquet';++statement ok+CREATE EXTERNAL TABLE lj_probe (a VARCHAR, b VARCHAR, e DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/lj_probe.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++# LEFT JOIN: build is preserved; probe rows get dynamic filter based on build keys.+query TT+EXPLAIN ANALYZE+SELECT lj_build.a, lj_build.b, lj_build.c, lj_probe.a, lj_probe.b, lj_probe.e+FROM lj_build LEFT JOIN lj_probe+ON lj_build.a = lj_probe.a AND lj_build.b = lj_probe.b;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=20.48% (214/1.04 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.78% (246/1.08 K)]++# LEFT SEMI JOIN: only matching build rows are returned; probe scan still+# receives the dynamic filter.+query TT+EXPLAIN ANALYZE+SELECT lj_build.*+FROM lj_build+WHERE EXISTS (+ SELECT 1 FROM lj_probe+ WHERE lj_probe.a = lj_build.a AND lj_probe.b = lj_build.b+);+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=4, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=20.48% (214/1.04 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.37% (166/1.08 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+drop table lj_build;++statement ok+drop table lj_probe;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_hashjoin_hash_table_pushdown_collect_left+# test_hashjoin_hash_table_pushdown_integer_keys+#+# With `hash_join_inlist_pushdown_max_size = 1`, the HashJoin dynamic filter+# should use the HashTable strategy (`hash_lookup`) instead of the InList+# strategy (`IN (SET) ([...])`), even when the build side is tiny.+########++statement ok+set datafusion.optimizer.hash_join_inlist_pushdown_max_size = 1;++# --- multi-column string keys ---++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0)+ ) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_parquet/hl_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ ('aa', 'ba', 1.0),+ ('ab', 'bb', 2.0),+ ('ac', 'bc', 3.0),+ ('ad', 'bd', 4.0)+ ) AS v(a, b, e)+) TO 'test_files/scratch/push_down_filter_parquet/hl_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE hl_build (a VARCHAR, b VARCHAR, c DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/hl_build.parquet';++statement ok+CREATE EXTERNAL TABLE hl_probe (a VARCHAR, b VARCHAR, e DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/hl_probe.parquet';++statement ok+set datafusion.explain.analyze_categories = 'rows';++# probe scan filter should contain `hash_lookup`, not `IN (SET)`.+query TT+EXPLAIN ANALYZE+SELECT p.a, p.b, build.c, p.e+FROM hl_probe p INNER JOIN hl_build AS build+ ON p.a = build.a AND p.b = build.b;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=20.48% (214/1.04 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND hash_lookup ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.78% (246/1.08 K)]++statement ok+drop table hl_build;++statement ok+drop table hl_probe;++# --- multi-column integer keys ---++statement ok+COPY (+ SELECT * FROM (VALUES+ (1, 10, 100.0),+ (2, 20, 200.0)+ ) AS v(id1, id2, value)+) TO 'test_files/scratch/push_down_filter_parquet/int_build.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES+ (1, 10, 'a'),+ (2, 20, 'b'),+ (3, 30, 'c'),+ (4, 40, 'd')+ ) AS v(id1, id2, data)+) TO 'test_files/scratch/push_down_filter_parquet/int_probe.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE int_build (id1 INT, id2 INT, value DOUBLE)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/int_build.parquet';++statement ok+CREATE EXTERNAL TABLE int_probe (id1 INT, id2 INT, data VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/int_probe.parquet';++query TT+EXPLAIN ANALYZE+SELECT b.id1, b.id2, b.value, p.data+FROM int_build b INNER JOIN int_probe p+ ON b.id1 = p.id1 AND b.id2 = p.id2;+----+Plan with Metrics+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id1@0, id1@0), (id2@1, id2@1)], projection=[id1@0, id2@1, value@2, data@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_build.parquet]]}, projection=[id1, id2, value], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.02% (222/1.17 K)]+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_probe.parquet]]}, projection=[id1, id2, data], file_type=parquet, predicate=DynamicFilter [ id1@0 >= 1 AND id1@0 <= 2 AND id2@1 >= 10 AND id2@1 <= 20 AND hash_lookup ], pruning_predicate=id1_null_count@1 != row_count@2 AND id1_max@0 >= 1 AND id1_null_count@1 != row_count@2 AND id1_min@3 <= 2 AND id2_null_count@5 != row_count@2 AND id2_max@4 >= 10 AND id2_null_count@5 != row_count@2 AND id2_min@6 <= 20, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=21.43% (239/1.11 K)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+reset datafusion.optimizer.hash_join_inlist_pushdown_max_size;++statement ok+drop table int_build;++statement ok+drop table int_probe;++# Config resetstatement okRESET datafusion.explain.physical_plan_only;diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex cfc564fa2b..51ad998c02 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -185,9 +185,318 @@ physical_plan04)------AggregateExec: mode=Partial, gby=[(NULL as id), (id@0 as id)], aggr=[max(agg_dyn_test.id)]05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-01/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-02/j5fUeSDQo22oPyPU.parquet], [WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-03/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-04/j5fUeSDQo22oPyPU.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 < 10, pruning_predicate=id_null_count@1 != row_count@2 AND id_min@0 < 10, required_guarantees=[]+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_aggregate_dynamic_filter_parquet_e2e+#+# End-to-end: `select max(id) from t where id > 1` should pick up a dynamic+# filter from the partial aggregate that prunes at least one file range+# (files_ranges_pruned_statistics must report < 4 matched out of 4 total).+#+# Build a self-contained fixture of 4 single-row parquet files, one per+# `id` value. With target_partitions=2, files are split into 2 groups of 2.+# Writing the files in decreasing-id order ensures the first file in each+# group has the larger id, so after reading it the dynamic filter tightens+# (e.g. to `id > 4` in group 1) and prunes the second file in the group.++statement ok+COPY (VALUES (4)) TO 'test_files/scratch/push_down_filter_regression/agg_dyn/file_0.parquet' STORED AS PARQUET;++statement ok+COPY (VALUES (3)) TO 'test_files/scratch/push_down_filter_regression/agg_dyn/file_1.parquet' STORED AS PARQUET;++statement ok+COPY (VALUES (2)) TO 'test_files/scratch/push_down_filter_regression/agg_dyn/file_2.parquet' STORED AS PARQUET;++statement ok+COPY (VALUES (1)) TO 'test_files/scratch/push_down_filter_regression/agg_dyn/file_3.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_dyn_e2e (column1 INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn/';++statement ok+set datafusion.execution.collect_statistics = true;++statement ok+set datafusion.explain.analyze_categories = 'rows';++query TT+EXPLAIN ANALYZE select max(column1) from agg_dyn_e2e where column1 > 1;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[output_rows=1, output_batches=1]+02)--CoalescePartitionsExec, metrics=[output_rows=2, output_batches=2]+03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_e2e.column1)], metrics=[output_rows=2, output_batches=2]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_2.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn/file_3.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 > 1 AND DynamicFilter [ column1@0 > 4 ], pruning_predicate=column1_null_count@1 != row_count@2 AND column1_max@0 > 1 AND column1_null_count@1 != row_count@2 AND column1_max@0 > 4, required_guarantees=[], metrics=[output_rows=2, output_batches=2, files_ranges_pruned_statistics=4 total → 4 matched, row_groups_pruned_statistics=4 total → 2 matched -> 2 fully matched, row_groups_pruned_bloom_filter=2 total → 2 matched, page_index_pages_pruned=2 total → 2 matched, page_index_rows_pruned=2 total → 2 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=4, files_processed=4, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=0, predicate_cache_inner_records=2, predicate_cache_records=4, scan_efficiency_ratio=25.15% (130/517)]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+reset datafusion.execution.collect_statistics;++statement ok+drop table agg_dyn_e2e;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs:+# test_aggregate_dynamic_filter_min_simple+# test_aggregate_dynamic_filter_max_simple+# test_aggregate_dynamic_filter_min_expression_not_supported+# test_aggregate_dynamic_filter_min_max_same_column+# test_aggregate_dynamic_filter_min_max_different_columns+# test_aggregate_dynamic_filter_multiple_mixed_expressions+# test_aggregate_dynamic_filter_min_all_nulls+#+# Baseline behavior of aggregate dynamic filter emission:+# MIN(a) -> filter `a < <min_seen>`+# MAX(a) -> filter `a > <max_seen>`+# MIN + MAX -> filter `a < <min> OR a > <max>` (or across columns)+# MIN(a+1) -> no dynamic filter (expression input is unsupported)+# all-NULLs -> filter stays `true` (no meaningful bound to propagate)+#+# Dynamic aggregate filters are only emitted in Partial+Final mode, not+# Single mode, so each fixture is written to 2 files and runs with+# target_partitions=2 to force a multi-partition aggregate plan.++statement ok+set datafusion.execution.target_partitions = 2;++# --- single-column fixture ([5, 1, 3, 8]) split across 2 files ---++statement ok+COPY (+ SELECT * FROM (VALUES (5), (1)) AS v(a)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_single/file_0.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES (3), (8)) AS v(a)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_single/file_1.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_dyn_single (a INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_single/';++# Use `analyze_level = summary` + `analyze_categories = 'none'` so metrics+# render empty; we only care that the `predicate=DynamicFilter [ ... ]` text+# matches. Pruning metrics here are subject to a parallel-execution race+# (the order in which Partial aggregates publish filter updates vs. when the+# scan reads each partition), so the filter *content* is deterministic but+# the pruning counts are not.+statement ok+set datafusion.explain.analyze_level = summary;++statement ok+set datafusion.explain.analyze_categories = 'none';++# MIN(a) -> DynamicFilter [ a < 1 ]+query TT+EXPLAIN ANALYZE SELECT MIN(a) FROM agg_dyn_single;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_single.a)], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_single.a)], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_1.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 1 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 1, required_guarantees=[], metrics=[]++# MAX(a) -> DynamicFilter [ a > 8 ]+query TT+EXPLAIN ANALYZE SELECT MAX(a) FROM agg_dyn_single;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_single.a)], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_single.a)], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_1.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 > 8 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 8, required_guarantees=[], metrics=[]++# MIN(a), MAX(a) -> DynamicFilter [ a < 1 OR a > 8 ]+query TT+EXPLAIN ANALYZE SELECT MIN(a), MAX(a) FROM agg_dyn_single;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_single.a), max(agg_dyn_single.a)], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_single.a), max(agg_dyn_single.a)], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_1.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 1 OR a@0 > 8 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 1 OR a_null_count@1 != row_count@2 AND a_max@3 > 8, required_guarantees=[], metrics=[]++# MIN(a+1) -> no dynamic filter (expression input is not a plain column)+query TT+EXPLAIN ANALYZE SELECT MIN(a + 1) FROM agg_dyn_single;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_single.a + Int64(1))], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_single.a + Int64(1))], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_single/file_1.parquet]]}, projection=[a], file_type=parquet, metrics=[]++statement ok+drop table agg_dyn_single;++# --- two-column fixture: MIN(a) + MAX(b) across columns ---++statement ok+COPY (+ SELECT * FROM (VALUES (5, 7), (1, 2)) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_two_col/file_0.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES (3, 4), (8, 9)) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_two_col/file_1.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_dyn_two_col (a INT, b INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_two_col/';++# MIN(a), MAX(b) -> DynamicFilter [ a < 1 OR b > 9 ]+query TT+EXPLAIN ANALYZE SELECT MIN(a), MAX(b) FROM agg_dyn_two_col;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_two_col.a), max(agg_dyn_two_col.b)], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_two_col.a), max(agg_dyn_two_col.b)], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_two_col/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_two_col/file_1.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 < 1 OR b@1 > 9 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 1 OR b_null_count@4 != row_count@2 AND b_max@3 > 9, required_guarantees=[], metrics=[]++statement ok+drop table agg_dyn_two_col;++# --- mixed expressions: MIN(a), MAX(a), MAX(b), MIN(c+1) ---+# Supported aggregates (MIN(a), MAX(a), MAX(b)) should drive a filter;+# MIN(c+1) is unsupported and must not contribute.++statement ok+COPY (+ SELECT * FROM (VALUES (5, 10, 100), (1, 4, 70)) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_mixed/file_0.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT * FROM (VALUES (3, 6, 90), (8, 12, 110)) AS v(a, b, c)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_mixed/file_1.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_dyn_mixed (a INT, b INT, c INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_mixed/';++# -> DynamicFilter [ a < 1 OR a > 8 OR b > 12 ] (MIN(c+1) dropped as unsupported)+query TT+EXPLAIN ANALYZE SELECT MIN(a), MAX(a), MAX(b), MIN(c + 1) FROM agg_dyn_mixed;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_mixed.a), max(agg_dyn_mixed.a), max(agg_dyn_mixed.b), min(agg_dyn_mixed.c + Int64(1))], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_mixed.a), max(agg_dyn_mixed.a), max(agg_dyn_mixed.b), min(agg_dyn_mixed.c + Int64(1))], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_mixed/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_mixed/file_1.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ a@0 < 1 OR a@0 > 8 OR b@1 > 12 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 1 OR a_null_count@1 != row_count@2 AND a_max@3 > 8 OR b_null_count@5 != row_count@2 AND b_max@4 > 12, required_guarantees=[], metrics=[]++statement ok+drop table agg_dyn_mixed;++# --- all-NULLs input: filter should stay `true` (no meaningful bound) ---++statement ok+COPY (+ SELECT CAST(NULL AS INT) AS a FROM (VALUES (1), (2)) AS v(n)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_nulls/file_0.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT CAST(NULL AS INT) AS a FROM (VALUES (3), (4)) AS v(n)+) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_nulls/file_1.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_dyn_nulls (a INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_nulls/';++# MIN(a) over all-NULL input -> DynamicFilter [ true ]+query TT+EXPLAIN ANALYZE SELECT MIN(a) FROM agg_dyn_nulls;+----+Plan with Metrics+01)AggregateExec: mode=Final, gby=[], aggr=[min(agg_dyn_nulls.a)], metrics=[]+02)--CoalescePartitionsExec, metrics=[]+03)----AggregateExec: mode=Partial, gby=[], aggr=[min(agg_dyn_nulls.a)], metrics=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_nulls/file_0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_dyn_nulls/file_1.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ true ], metrics=[]++statement ok+reset datafusion.explain.analyze_categories;++statement ok+reset datafusion.explain.analyze_level;++statement ok+drop table agg_dyn_nulls;++statement ok+set datafusion.execution.target_partitions = 4;+++########+# Ported from datafusion/core/tests/physical_optimizer/filter_pushdown.rs::+# test_aggregate_filter_pushdown+# test_no_pushdown_filter_on_aggregate_result+#+# - A filter that only references grouping columns (`WHERE a = 'x' GROUP BY a`)+# should push through the aggregate and land on the scan's predicate.+# - A filter on an aggregate *result* (`HAVING count(b) > 5`) must NOT push+# through — it has to stay above the aggregate.+########++statement ok+COPY (+ SELECT * FROM (VALUES ('x', 'foo'), ('y', 'bar')) AS v(a, b)+) TO 'test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE agg_filter_pushdown (a VARCHAR, b VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet';++# Filter on grouping column pushes down to the scan.+query TT+EXPLAIN SELECT a, count(b) FROM agg_filter_pushdown WHERE a = 'x' GROUP BY a;+----+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted+02)--RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4+03)----AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)], ordering_mode=Sorted+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet, predicate=a@0 = x, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= x AND x <= a_max@1, required_guarantees=[a in (x)]++# Filter on aggregate result (HAVING) stays above the aggregate.+query TT+EXPLAIN SELECT a, count(b) AS cnt FROM agg_filter_pushdown GROUP BY a HAVING count(b) > 5;+----+physical_plan+01)ProjectionExec: expr=[a@0 as a, count(agg_filter_pushdown.b)@1 as cnt]+02)--FilterExec: count(agg_filter_pushdown.b)@1 > 5+03)----AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]+04)------RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1+05)--------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)]+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet++statement ok+drop table agg_filter_pushdown;+# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okset datafusion.execution.target_partitions = 4;