Apache DataFusion SQL Query Engine
Enable dynamic filters for range-partitioned joins (#23854)
Peter Lee618aaff82158
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex a26761107a..a98c1b7bcf 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -18,7 +18,7 @@use std::sync::{Arc, LazyLock};use arrow::{- array::record_batch,+ array::{RecordBatch, record_batch},datatypes::{DataType, Field, Schema, SchemaRef},util::pretty::pretty_format_batches,};@@ -55,7 +55,8 @@ use datafusion_physical_expr::{utils::conjunction,};use datafusion_physical_expr::{- Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder,+ Partitioning, RangePartitioning, ScalarFunctionExpr, SplitPoint,+ aggregate::AggregateExprBuilder,};use datafusion_physical_optimizer::{PhysicalOptimizerRule, filter_pushdown::FilterPushdown,@@ -187,9 +188,6 @@ fn test_pushdown_into_scan_with_config_options() {// distinction this test exercises is not reachable via SQL.#[tokio::test]async fn test_static_filter_pushdown_through_hash_join() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-// Create build side with limited valueslet build_batches = vec![record_batch!(@@ -945,15 +943,73 @@ async fn test_topk_filter_passes_through_coalesce_partitions() {);}+fn hashjoin_pushdown_scans() -> (+ SchemaRef,+ Arc<dyn ExecutionPlan>,+ SchemaRef,+ Arc<dyn ExecutionPlan>,+) {+ let build_side_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Utf8, false),+ Field::new("c", DataType::Float64, false),+ ]));+ let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))+ .with_support(true)+ .with_batches(vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab"]),+ ("b", Utf8, ["ba", "bb"]),+ ("c", Float64, [1.0, 2.0])+ )+ .unwrap(),+ ])+ .build();++ let probe_side_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Utf8, false),+ Field::new("e", DataType::Float64, false),+ ]));+ let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))+ .with_support(true)+ .with_batches(vec![+ record_batch!(+ ("a", Utf8, ["aa", "ab", "ac", "ad"]),+ ("b", Utf8, ["ba", "bb", "bc", "bd"]),+ ("e", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ])+ .build();++ (build_side_schema, build_scan, probe_side_schema, probe_scan)+}++async fn optimize_and_collect_pushdown_plan(+ plan: Arc<dyn ExecutionPlan>,+ config: ConfigOptions,+) -> (Arc<dyn ExecutionPlan>, Vec<RecordBatch>) {+ let plan = FilterPushdown::new_post_optimization()+ .optimize(plan, &config)+ .unwrap();+ let session_ctx =+ SessionContext::new_with_config(SessionConfig::from(config).with_batch_size(10));+ session_ctx.register_object_store(+ ObjectStoreUrl::parse("test://").unwrap().as_ref(),+ Arc::new(InMemory::new()),+ );+ let task_ctx = session_ctx.state().task_ctx();+ let batches = collect(Arc::clone(&plan), task_ctx).await.unwrap();+ (plan, batches)+}+// Not portable to sqllogictest: this test pins `PartitionMode::Partitioned`// by hand-wiring `RepartitionExec(Hash, 12)` on both join sides. A SQL// INNER JOIN over small parquet inputs plans as `CollectLeft`, so the// per-partition CASE filter this test exercises is not reachable via SQL.#[tokio::test]async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-// Rough sketch of the MRE we're trying to recreate:// COPY (select i as k from generate_series(1, 10000000) as t(i))// TO 'test_files/scratch/push_down_filter/t1.parquet'@@ -994,43 +1050,8 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {// | | |// +---------------+------------------------------------------------------------+- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();+ let (build_side_schema, build_scan, probe_side_schema, probe_scan) =+ hashjoin_pushdown_scans();// Create RepartitionExec nodes for both sides with hash partitioning on join keyslet partition_count = 12;@@ -1128,20 +1149,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;// Now check what our filter looks like#[cfg(not(feature = "force_hash_collisions"))]@@ -1198,53 +1206,214 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() {);}-// Not portable to sqllogictest: this test specifically pins a-// `RepartitionExec(Hash, 12)` between `HashJoinExec(CollectLeft)` and the-// probe-side scan to verify the dynamic filter link survives that boundary-// (regression for #17451). The same CollectLeft filter content and-// pushdown counters are already covered by the simpler slt port-// (push_down_filter_parquet.slt::test_hashjoin_dynamic_filter_pushdown).#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};+async fn test_hashjoin_dynamic_filter_pushdown_range_partitioned() {+ // Rough sketch of the Range-partitioned MRE we're trying to recreate. The+ // test hand-wires identical Range repartitioning:+ //+ // EXPLAIN+ // SELECT *+ // FROM build+ // JOIN probe+ // ON build.a = probe.a AND build.b = probe.b;+ //+ // +---------------+------------------------------------------------------------++ // | plan_type | plan |+ // +---------------+------------------------------------------------------------++ // | physical_plan | ┌───────────────────────────┐ |+ // | | │ HashJoinExec │ |+ // | | │ -------------------- ├──────────────┐ |+ // | | │ on: (a = a), (b = b) │ │ |+ // | | └─────────────┬─────────────┘ │ |+ // | | ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ |+ // | | │ RepartitionExec ││ RepartitionExec │ |+ // | | │ -------------------- ││ -------------------- │ |+ // | | │ partition_count(in->out): ││ partition_count(in->out): │ |+ // | | │ 1 -> 2 ││ 1 -> 2 │ |+ // | | │ ││ │ |+ // | | │ partitioning_scheme: ││ partitioning_scheme: │ |+ // | | │ Range([a ASC, b ASC], 2) ││ Range([a ASC, b ASC], 2) │ |+ // | | │ split: (aa, bb) ││ split: (aa, bb) │ |+ // | | └─────────────┬─────────────┘└─────────────┬─────────────┘ |+ // | | ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ |+ // | | │ DataSourceExec (build) ││ DataSourceExec (probe) │ |+ // | | │ -------------------- ││ -------------------- │ |+ // | | │ rows: (aa,ba), (ab,bb) ││ rows: (aa,ba) ... (ad,bd) │ |+ // | | │ ││ predicate: DynamicFilter │ |+ // | | │ ││ range CASE -> filter_0/1 │ |+ // | | └───────────────────────────┘└───────────────────────────┘ |+ // | | |+ // +---------------+------------------------------------------------------------+- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join+ let (build_side_schema, build_scan, probe_side_schema, probe_scan) =+ hashjoin_pushdown_scans();++ let split_points = vec![SplitPoint::new(vec![+ ScalarValue::Utf8(Some("aa".to_string())),+ ScalarValue::Utf8(Some("bb".to_string())),+ ])];++ // Build side: DataSource -> RepartitionExec (Range)+ let build_range_ordering = LexOrdering::new(vec![+ PhysicalSortExpr::new(+ col("a", &build_side_schema).unwrap(),+ SortOptions::default(),+ ),+ PhysicalSortExpr::new(+ col("b", &build_side_schema).unwrap(),+ SortOptions::default(),+ ),+ ])+ .unwrap();+ let build_repartition = Arc::new(+ RepartitionExec::try_new(+ build_scan,+ Partitioning::Range(+ RangePartitioning::try_new(build_range_ordering, split_points.clone())+ .unwrap(),+ ),).unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();+ );- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join+ // Probe side: DataSource -> RepartitionExec (Range)+ let probe_range_ordering = LexOrdering::new(vec![+ PhysicalSortExpr::new(+ col("a", &probe_side_schema).unwrap(),+ SortOptions::default(),+ ),+ PhysicalSortExpr::new(+ col("b", &probe_side_schema).unwrap(),+ SortOptions::default(),+ ),+ ])+ .unwrap();+ let probe_repartition = Arc::new(+ RepartitionExec::try_new(+ Arc::clone(&probe_scan),+ Partitioning::Range(+ RangePartitioning::try_new(probe_range_ordering, split_points).unwrap(),+ ),).unwrap(),+ );++ // Create HashJoinExec with partitioned inputs+ let on = vec![+ (+ col("a", &build_side_schema).unwrap(),+ col("a", &probe_side_schema).unwrap(),+ ),+ (+ col("b", &build_side_schema).unwrap(),+ col("b", &probe_side_schema).unwrap(),+ ),];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();+ let hash_join = Arc::new(+ HashJoinExec::try_new(+ build_repartition,+ probe_repartition,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::Partitioned,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ );++ // Top-level CoalescePartitionsExec+ let cp = Arc::new(CoalescePartitionsExec::new(hash_join)) as Arc<dyn ExecutionPlan>;+ // Add a sort for deterministic output+ let plan = Arc::new(SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr::new(+ col("a", &probe_side_schema).unwrap(),+ SortOptions::new(true, false), // descending, nulls_first+ )])+ .unwrap(),+ cp,+ )) as Arc<dyn ExecutionPlan>;++ // expect the predicate to be pushed down into the probe side DataSource+ insta::assert_snapshot!(+ OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ]+ "+ );++ // Actually apply the optimization to the plan and execute to see the filter in action+ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_dynamic_filter_pushdown = true;+ config.optimizer.preserve_file_partitions = 1;+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;++ // Now check what our filter looks like+ insta::assert_snapshot!(+ format!("{}", format_plan_for_test(&plan)),+ @r"+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true+ - RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(aa, bb)], 2), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, e], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN a@0 >= aa AND a@0 <= aa AND b@1 >= ba AND b@1 <= ba AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}]) ELSE a@0 >= ab AND a@0 <= ab AND b@1 >= bb AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:ab,c1:bb}]) END ]+ "+ );++ let result = format!("{}", pretty_format_batches(&batches).unwrap());++ let probe_scan_metrics = probe_scan.metrics().unwrap();++ // The probe side had 4 rows, but after applying the dynamic filter only 2 rows should remain.+ // The number of output rows from the probe side scan should stay consistent across executions.+ // Issue: https://github.com/apache/datafusion/issues/17451+ assert_eq!(probe_scan_metrics.output_rows().unwrap(), 2);++ insta::assert_snapshot!(+ result,+ @r"+ +----+----+-----+----+----+-----++ | a | b | c | a | b | e |+ +----+----+-----+----+----+-----++ | ab | bb | 2.0 | ab | bb | 2.0 |+ | aa | ba | 1.0 | aa | ba | 1.0 |+ +----+----+-----+----+----+-----++ ",+ );+}++// Not portable to sqllogictest: this test specifically pins a+// `RepartitionExec(Hash, 12)` between `HashJoinExec(CollectLeft)` and the+// probe-side scan to verify the dynamic filter link survives that boundary+// (regression for #17451). The same CollectLeft filter content and+// pushdown counters are already covered by the simpler slt port+// (push_down_filter_parquet.slt::test_hashjoin_dynamic_filter_pushdown).+#[tokio::test]+async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {+ let (build_side_schema, build_scan, probe_side_schema, probe_scan) =+ hashjoin_pushdown_scans();// Create RepartitionExec nodes for both sides with hash partitioning on join keyslet partition_count = 12;@@ -1326,20 +1495,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {let mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, &config)- .unwrap();- let config = SessionConfig::new().with_batch_size(10);- let session_ctx = SessionContext::new_with_config(config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;// Now check what our filter looks likeinsta::assert_snapshot!(@@ -1378,9 +1534,6 @@ async fn test_hashjoin_dynamic_filter_pushdown_collect_left() {#[test]fn test_hashjoin_parent_filter_pushdown_same_column_names() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-let build_side_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false),Field::new("build_val", DataType::Utf8, false),@@ -1447,9 +1600,6 @@ fn test_hashjoin_parent_filter_pushdown_same_column_names() {#[test]fn test_hashjoin_parent_filter_pushdown_mark_join() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-let left_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false),Field::new("val", DataType::Utf8, false),@@ -1517,9 +1667,6 @@ fn test_hashjoin_parent_filter_pushdown_mark_join() {/// only rely on the output side to preserve their semantics.#[test]fn test_hashjoin_parent_filter_pushdown_semi_anti_join() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-let left_schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Utf8, false),Field::new("v", DataType::Utf8, false),@@ -2475,9 +2622,6 @@ fn test_pushdown_with_computed_grouping_key() {// on a hand-wired plan, which does trigger the `false` path.#[tokio::test]async fn test_hashjoin_dynamic_filter_all_partitions_empty() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-// Test scenario where all build-side partitions are empty// This validates the code path that sets the filter to `false` when no rows can match@@ -2610,46 +2754,8 @@ async fn test_hashjoin_dynamic_filter_all_partitions_empty() {// PartitionMode::Partitioned, which SQL never picks for small parquet inputs.#[tokio::test]async fn test_hashjoin_hash_table_pushdown_partitioned() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- // Create build side with limited values- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();+ let (build_side_schema, build_scan, probe_side_schema, probe_scan) =+ hashjoin_pushdown_scans();// Create RepartitionExec nodes for both sides with hash partitioning on join keyslet partition_count = 12;@@ -2719,24 +2825,11 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() {)) as Arc<dyn ExecutionPlan>;// Apply the optimization with config setting that forces HashTable strategy- let session_config = SessionConfig::default()- .with_batch_size(10)- .set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1)- .set_bool("datafusion.execution.parquet.pushdown_filters", true)- .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true);- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, session_config.options())- .unwrap();- let session_ctx = SessionContext::new_with_config(session_config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();+ let mut config = ConfigOptions::default();+ config.optimizer.hash_join_inlist_pushdown_max_size = 1;+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_dynamic_filter_pushdown = true;+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;// Verify that hash_lookup is used instead of IN (SET)let plan_str = format_plan_for_test(&plan).to_string();@@ -2776,45 +2869,8 @@ async fn test_hashjoin_hash_table_pushdown_partitioned() {// IN (SET) invariant is captured in the slt port.#[tokio::test]async fn test_hashjoin_hash_table_pushdown_collect_left() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};-- let build_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab"]),- ("b", Utf8, ["ba", "bb"]),- ("c", Float64, [1.0, 2.0]) // Extra column not used in join- )- .unwrap(),- ];- let build_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("c", DataType::Float64, false),- ]));- let build_scan = TestScanBuilder::new(Arc::clone(&build_side_schema))- .with_support(true)- .with_batches(build_batches)- .build();-- // Create probe side with more values- let probe_batches = vec![- record_batch!(- ("a", Utf8, ["aa", "ab", "ac", "ad"]),- ("b", Utf8, ["ba", "bb", "bc", "bd"]),- ("e", Float64, [1.0, 2.0, 3.0, 4.0]) // Extra column not used in join- )- .unwrap(),- ];- let probe_side_schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Utf8, false),- Field::new("b", DataType::Utf8, false),- Field::new("e", DataType::Float64, false),- ]));- let probe_scan = TestScanBuilder::new(Arc::clone(&probe_side_schema))- .with_support(true)- .with_batches(probe_batches)- .build();+ let (build_side_schema, build_scan, probe_side_schema, probe_scan) =+ hashjoin_pushdown_scans();// Create RepartitionExec nodes for both sides with hash partitioning on join keyslet partition_count = 12;@@ -2870,24 +2926,11 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {)) as Arc<dyn ExecutionPlan>;// Apply the optimization with config setting that forces HashTable strategy- let session_config = SessionConfig::default()- .with_batch_size(10)- .set_usize("datafusion.optimizer.hash_join_inlist_pushdown_max_size", 1)- .set_bool("datafusion.execution.parquet.pushdown_filters", true)- .set_bool("datafusion.optimizer.enable_dynamic_filter_pushdown", true);- let plan = FilterPushdown::new_post_optimization()- .optimize(plan, session_config.options())- .unwrap();- let session_ctx = SessionContext::new_with_config(session_config);- session_ctx.register_object_store(- ObjectStoreUrl::parse("test://").unwrap().as_ref(),- Arc::new(InMemory::new()),- );- let state = session_ctx.state();- let task_ctx = state.task_ctx();- let batches = collect(Arc::clone(&plan), Arc::clone(&task_ctx))- .await- .unwrap();+ let mut config = ConfigOptions::default();+ config.optimizer.hash_join_inlist_pushdown_max_size = 1;+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_dynamic_filter_pushdown = true;+ let (plan, batches) = optimize_and_collect_pushdown_plan(plan, config).await;// Verify that hash_lookup is used instead of IN (SET)let plan_str = format_plan_for_test(&plan).to_string();@@ -3620,7 +3663,6 @@ fn test_filter_pushdown_through_sort_with_projection() {#[test]fn post_phase_is_idempotent_on_hash_join() {use crate::physical_optimizer::test_utils::{hash_join_exec, parquet_exec, schema};- use datafusion_common::JoinType;use datafusion_physical_expr::expressions::Column;use datafusion_physical_optimizer::filter_pushdown::FilterPushdown;use datafusion_physical_plan::get_plan_string;diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 4e68d871b8..cd9048b58c 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -876,14 +876,24 @@ impl HashJoinExec {return false;}- // `preserve_file_partitions` can report Hash partitioning for Hive-style- // file groups, but those partitions are not actually hash-distributed.- // Partitioned dynamic filters rely on hash routing, so disable them in- // this mode to avoid incorrect results. Follow-up work: enable dynamic- // filtering for preserve_file_partitioned scans (issue #20195).+ // `preserve_file_partitions` can report Hive-style file groups as Hash+ // partitioned even though their partition indexes do not follow the+ // hash router used by partitioned dynamic filters. Reject Hash inputs+ // because the metadata cannot distinguish those scans from a real hash+ // repartition. Compatible Range inputs remain safe because matching+ // ordering and split points align each build filter with its probe+ // partition. Other unsupported layouts are rejected.+ // Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195).// https://github.com/apache/datafusion/issues/20195if config.optimizer.preserve_file_partitions > 0&& self.mode == PartitionMode::Partitioned+ && matches!(+ (+ self.left.output_partitioning(),+ self.right.output_partitioning()+ ),+ (Partitioning::Hash(_, _), Partitioning::Hash(_, _))+ ){return false;}@@ -891,9 +901,6 @@ impl HashJoinExec {if self.mode == PartitionMode::Partitioned&& !self.has_partitioned_dynamic_filter_routing(){- // TODO: support partition-routed dynamic filters for compatible- // range co-partitioned joins.- // <https://github.com/apache/datafusion/issues/23376>.return false;}@@ -909,6 +916,14 @@ impl HashJoinExec {Partitioning::Hash(_, left_partition_count),Partitioning::Hash(_, right_partition_count),) => left_partition_count == right_partition_count,+ (Partitioning::Range(_), Partitioning::Range(_)) => {+ let children = [self.left.as_ref(), self.right.as_ref()];+ matches!(+ self.input_distribution_requirements()+ .unsatisfied_co_partitioned_children(self.name(), &children),+ Ok(unsatisfied) if unsatisfied.is_empty()+ )+ }(left_partitioning, right_partitioning) => {left_partitioning.partition_count() == 1&& right_partitioning.partition_count() == 1@@ -7089,9 +7104,10 @@ mod tests {Ok(())}- #[test]- fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()>- {+ fn range_partitioned_dynamic_filter_test_join(+ left_split: i32,+ right_split: i32,+ ) -> Result<(HashJoinExec, JoinOn)> {let (left_schema, right_schema, on) = build_schema_and_on()?;let left_partitioning = Partitioning::Range(RangePartitioning::try_new([PhysicalSortExpr {@@ -7099,7 +7115,7 @@ mod tests {options: Default::default(),}].into(),- vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(left_split))])],)?);let right_partitioning = Partitioning::Range(RangePartitioning::try_new([PhysicalSortExpr {@@ -7107,7 +7123,7 @@ mod tests {options: Default::default(),}].into(),- vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(right_split))])],)?);let left = Arc::new(PartitionedTestExec::try_new(left_schema,@@ -7118,16 +7134,10 @@ mod tests {right_partitioning,)?);- let mut session_config = SessionConfig::default();- session_config- .options_mut()- .optimizer- .enable_join_dynamic_filter_pushdown = true;-let join = HashJoinExec::try_new(left,right,- on,+ on.clone(),None,&JoinType::Inner,None,@@ -7135,8 +7145,73 @@ mod tests {NullEquality::NullEqualsNothing,false,)?;+ Ok((join, on))+ }- assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options()));+ fn with_hash_partitioned_children(+ join: &HashJoinExec,+ on: &JoinOn,+ ) -> Result<HashJoinExec> {+ join.builder()+ .with_new_children(vec![+ Arc::new(PartitionedTestExec::try_new(+ join.left().schema(),+ Partitioning::Hash(vec![Arc::clone(&on[0].0)], 2),+ )?),+ Arc::new(PartitionedTestExec::try_new(+ join.right().schema(),+ Partitioning::Hash(vec![Arc::clone(&on[0].1)], 2),+ )?),+ ])?+ .build()+ }++ #[test]+ fn test_partitioned_dynamic_filter_pushdown_allows_supported_partitioning()+ -> Result<()> {+ let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;+ let hash_join = with_hash_partitioned_children(&range_join, &on)?;+ let mut session_config = SessionConfig::default();+ session_config+ .options_mut()+ .optimizer+ .enable_join_dynamic_filter_pushdown = true;++ assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));+ assert!(hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));++ session_config+ .options_mut()+ .optimizer+ .preserve_file_partitions = 1;+ assert!(range_join.allow_join_dynamic_filter_pushdown(session_config.options()));++ Ok(())+ }++ #[test]+ fn test_partitioned_dynamic_filter_pushdown_rejects_unsupported_partitioning()+ -> Result<()> {+ let (range_join, on) = range_partitioned_dynamic_filter_test_join(10, 10)?;+ let hash_join = with_hash_partitioned_children(&range_join, &on)?;+ let (mismatched_range_join, _) =+ range_partitioned_dynamic_filter_test_join(10, 11)?;+ let mut session_config = SessionConfig::default();+ session_config+ .options_mut()+ .optimizer+ .enable_join_dynamic_filter_pushdown = true;++ assert!(+ !mismatched_range_join+ .allow_join_dynamic_filter_pushdown(session_config.options())+ );++ session_config+ .options_mut()+ .optimizer+ .preserve_file_partitions = 1;+ assert!(!hash_join.allow_join_dynamic_filter_pushdown(session_config.options()));Ok(())}diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 7b58107e93..94ec4565a4 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -23,6 +23,7 @@ use std::sync::Arc;use crate::ExecutionPlan;use crate::ExecutionPlanProperties;+use crate::Partitioning;use crate::joins::Map;use crate::joins::PartitionMode;use crate::joins::hash_join::exec::HASH_JOIN_SEED;@@ -30,18 +31,22 @@ use crate::joins::hash_join::inlist_builder::build_struct_fields;use crate::joins::hash_join::partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState,};+use crate::repartition::RangeExpr;use arrow::array::ArrayRef;use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::config::ConfigOptions;use datafusion_common::{DataFusionError, NullEquality, Result, ScalarValue, SharedResult,+ assert_or_internal_err,};use datafusion_expr::Operator;use datafusion_functions::core::r#struct as struct_func;use datafusion_physical_expr::expressions::{BinaryExpr, CaseExpr, DynamicFilterPhysicalExpr, InListExpr, IsNullExpr, lit,};-use datafusion_physical_expr::{PhysicalExpr, PhysicalExprRef, ScalarFunctionExpr};+use datafusion_physical_expr::{+ PhysicalExpr, PhysicalExprRef, RangePartitioning, ScalarFunctionExpr,+};use parking_lot::Mutex;use tokio::sync::Notify;@@ -257,6 +262,8 @@ pub(crate) struct SharedBuildAccumulator {repartition_random_state: SeededRandomState,/// Schema of the probe (right) side for evaluating filter expressionsprobe_schema: Arc<Schema>,+ /// Probe-side Range routing metadata for partitioned dynamic filters.+ probe_range_partitioning: Option<RangePartitioning>,/// Null equality of the join. Under `NullEqualsNull` a probe-side NULL can match a/// build-side NULL, so the pushed filter must keep NULL rows here too.null_equality: NullEquality,@@ -410,6 +417,14 @@ impl SharedBuildAccumulator {),};+ let probe_range_partitioning =+ match (partition_mode, right_child.output_partitioning()) {+ (PartitionMode::Partitioned, Partitioning::Range(range)) => {+ Some(range.clone())+ }+ _ => None,+ };+Self {inner: Mutex::new(AccumulatorState {data: mode_data,@@ -420,6 +435,7 @@ impl SharedBuildAccumulator {on_right,repartition_random_state,probe_schema: right_child.schema(),+ probe_range_partitioning,null_equality,null_aware,}@@ -628,19 +644,8 @@ impl SharedBuildAccumulator {},FinalizeInput::Partitioned(partitions) => {let num_partitions = partitions.len();- let routing_hash_expr = Arc::new(HashExpr::new(- self.on_right.clone(),- self.repartition_random_state.clone(),- "hash_repartition".to_string(),- )) as Arc<dyn PhysicalExpr>;-- let modulo_expr = Arc::new(BinaryExpr::new(- routing_hash_expr,- Operator::Modulo,- lit(ScalarValue::UInt64(Some(num_partitions as u64))),- )) as Arc<dyn PhysicalExpr>;-- let mut real_branches = Vec::new();+ let mut partition_filters = Vec::with_capacity(num_partitions);+ let mut real_partition_ids = Vec::new();let mut empty_partition_ids = Vec::new();let mut has_canceled_unknown = false;let mut keys_have_null = false;@@ -651,8 +656,10 @@ impl SharedBuildAccumulator {if matches!(partition.pushdown, PushdownStrategy::Empty) =>{empty_partition_ids.push(partition_id);+ partition_filters.push(lit(false));}PartitionStatus::Reported(partition) => {+ real_partition_ids.push(partition_id);keys_have_null |= partition.keys_have_null;let membership_expr = create_membership_predicate(&self.on_right,@@ -669,13 +676,11 @@ impl SharedBuildAccumulator {bounds_expr,).unwrap_or_else(|| lit(true));- real_branches.push((- lit(ScalarValue::UInt64(Some(partition_id as u64))),- then_expr,- ));+ partition_filters.push(then_expr);}PartitionStatus::CanceledUnknown => {has_canceled_unknown = true;+ partition_filters.push(lit(true));// A canceled partition's build content is unknown, so it// may hold a NULL key.keys_have_null = true;@@ -688,38 +693,97 @@ impl SharedBuildAccumulator {}}- let filter_expr = if has_canceled_unknown {- let mut when_then_branches = empty_partition_ids+ let filter_expr = if has_canceled_unknown+ && real_partition_ids.is_empty()+ && empty_partition_ids.is_empty()+ {+ lit(true)+ } else if !has_canceled_unknown && real_partition_ids.is_empty() {+ lit(false)+ } else if !has_canceled_unknown+ && real_partition_ids.len() == 1+ && empty_partition_ids.len() + 1 == num_partitions+ {+ Arc::clone(&partition_filters[real_partition_ids[0]])+ } else if let Some(range_partitioning) = &self.probe_range_partitioning {+ // Range partitioning+ assert_or_internal_err!(+ partition_filters.len() == range_partitioning.partition_count(),+ "Dynamic filter partition count {} does not match Range partition count {}",+ partition_filters.len(),+ range_partitioning.partition_count()+ );+ let routing_range_expr = Arc::new(RangeExpr::try_new(+ self.on_right.clone(),+ range_partitioning,+ )?)+ as Arc<dyn PhysicalExpr>;+ let else_expr = partition_filters+ .pop()+ .expect("Range partitioning always has at least one partition");++ // CASE range_partition(key)+ // WHEN 0 THEN F0+ // WHEN 1 THEN F1+ // ...+ // ELSE Fn+ // END+ let when_then_expr = partition_filters.into_iter()- .map(|partition_id| {+ .enumerate()+ .map(|(partition_id, then_expr)| {(lit(ScalarValue::UInt64(Some(partition_id as u64))),- lit(false),+ then_expr,)})- .collect::<Vec<_>>();- when_then_branches.extend(real_branches);+ .collect();- if when_then_branches.is_empty() {- lit(true)- } else {- Arc::new(CaseExpr::try_new(- Some(modulo_expr),- when_then_branches,- Some(lit(true)),- )?) as Arc<dyn PhysicalExpr>- }- } else if real_branches.is_empty() {- lit(false)- } else if real_branches.len() == 1- && empty_partition_ids.len() + 1 == num_partitions- {- Arc::clone(&real_branches[0].1)+ Arc::new(CaseExpr::try_new(+ Some(routing_range_expr),+ when_then_expr,+ Some(else_expr),+ )?) as Arc<dyn PhysicalExpr>} else {+ // Hash partitioning+ let routing_hash_expr = Arc::new(HashExpr::new(+ self.on_right.clone(),+ self.repartition_random_state.clone(),+ "hash_repartition".to_string(),+ ))+ as Arc<dyn PhysicalExpr>;+ let modulo_expr = Arc::new(BinaryExpr::new(+ routing_hash_expr,+ Operator::Modulo,+ lit(ScalarValue::UInt64(Some(num_partitions as u64))),+ )) as Arc<dyn PhysicalExpr>;++ let mut when_then_branches = if has_canceled_unknown {+ empty_partition_ids+ .into_iter()+ .map(|partition_id| {+ (+ lit(ScalarValue::UInt64(Some(partition_id as u64))),+ lit(false),+ )+ })+ .collect::<Vec<_>>()+ } else {+ vec![]+ };+ when_then_branches.extend(real_partition_ids.into_iter().map(+ |partition_id| {+ (+ lit(ScalarValue::UInt64(Some(partition_id as u64))),+ Arc::clone(&partition_filters[partition_id]),+ )+ },+ ));+Arc::new(CaseExpr::try_new(Some(modulo_expr),- real_branches,- Some(lit(false)),+ when_then_branches,+ Some(lit(has_canceled_unknown)),)?) as Arc<dyn PhysicalExpr>};@@ -809,6 +873,7 @@ pub(super) fn make_partitioned_accumulator_for_test(on_right: vec![],repartition_random_state: SeededRandomState::with_seed(1),probe_schema,+ probe_range_partitioning: None,null_equality: NullEquality::NullEqualsNothing,null_aware: false,}@@ -831,8 +896,14 @@ pub(super) fn completed_partitions_for_test(acc: &SharedBuildAccumulator) -> usimod tests {use super::*;- use arrow::array::{ArrayRef, Int32Array};- use datafusion_physical_expr::expressions::{Column, Literal};+ use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int32Array};+ use arrow::compute::SortOptions;+ use arrow::record_batch::RecordBatch;+ use datafusion_common::SplitPoint;+ use datafusion_physical_expr::{+ PhysicalSortExpr,+ expressions::{Column, Literal},+ };fn test_on_right() -> Vec<PhysicalExprRef> {vec![Arc::new(Column::new("probe_key", 0))]@@ -867,6 +938,7 @@ mod tests {on_right,repartition_random_state: SeededRandomState::with_seed(1),probe_schema: test_probe_schema(),+ probe_range_partitioning: None,null_equality: NullEquality::NullEqualsNothing,null_aware: false,}@@ -1076,6 +1148,198 @@ mod tests {);}+ #[test]+ fn partitioned_range_dynamic_filter_routes_with_range_expr() -> Result<()> {+ let mut acc = make_partitioned_expr_accumulator_for_test(4);+ acc.probe_range_partitioning = Some(RangePartitioning::try_new(+ [PhysicalSortExpr::new(+ Arc::clone(&acc.on_right[0]),+ Default::default(),+ )]+ .into(),+ vec![+ SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(20))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),+ ],+ )?);++ acc.build_filter(FinalizeInput::Partitioned(vec![+ reported(PushdownStrategy::Empty, no_bounds()),+ PartitionStatus::CanceledUnknown,+ reported(in_list(&[20, 29]), no_bounds()),+ reported(in_list(&[30]), no_bounds()),+ ]))?;++ let expr = current_expr(&acc);+ let case = case_expr(&expr);+ assert!(+ case.expr()+ .and_then(|expr| expr.downcast_ref::<RangeExpr>())+ .is_some(),+ "Range routing must use RangeExpr"+ );+ assert_eq!(case.when_then_expr().len(), 3);++ let batch = RecordBatch::try_new(+ test_probe_schema(),+ vec![Arc::new(Int32Array::from(vec![+ 9, 10, 19, 20, 21, 29, 30, 31,+ ]))],+ )?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = result+ .as_any()+ .downcast_ref::<BooleanArray>()+ .expect("dynamic filter should evaluate to BooleanArray");+ assert_eq!(+ result,+ &BooleanArray::from(vec![false, true, true, true, false, true, true, false,])+ );++ Ok(())+ }++ #[test]+ fn partitioned_range_dynamic_filter_routes_compound_nullable_keys() -> Result<()> {+ let probe_schema = Arc::new(Schema::new(vec![+ Field::new("probe_key", DataType::Int32, true),+ Field::new("probe_tie", DataType::Int32, true),+ ]));+ let on_right: Vec<PhysicalExprRef> = vec![+ Arc::new(Column::new("probe_key", 0)),+ Arc::new(Column::new("probe_tie", 1)),+ ];+ let mut acc = make_accumulator_for_test(+ AccumulatedBuildData::Partitioned {+ partitions: vec![PartitionStatus::Pending; 4],+ completed_partitions: 0,+ },+ on_right,+ );+ acc.probe_schema = Arc::clone(&probe_schema);+ acc.probe_range_partitioning = Some(RangePartitioning::try_new(+ [+ PhysicalSortExpr::new(+ Arc::clone(&acc.on_right[0]),+ SortOptions::new(false, true),+ ),+ PhysicalSortExpr::new(+ Arc::clone(&acc.on_right[1]),+ SortOptions::new(false, false),+ ),+ ]+ .into(),+ vec![+ SplitPoint::new(vec![+ ScalarValue::Int32(None),+ ScalarValue::Int32(Some(10)),+ ]),+ SplitPoint::new(vec![ScalarValue::Int32(None), ScalarValue::Int32(None)]),+ SplitPoint::new(vec![+ ScalarValue::Int32(Some(10)),+ ScalarValue::Int32(None),+ ]),+ ],+ )?);++ acc.build_filter(FinalizeInput::Partitioned(vec![+ reported(PushdownStrategy::Empty, no_bounds()),+ PartitionStatus::CanceledUnknown,+ reported(PushdownStrategy::Empty, no_bounds()),+ PartitionStatus::CanceledUnknown,+ ]))?;++ let expr = current_expr(&acc);+ let case = case_expr(&expr);+ assert!(case.expr().is_some());+ assert_eq!(case.when_then_expr().len(), 3);++ let batch = RecordBatch::try_new(+ probe_schema,+ vec![+ Arc::new(Int32Array::from(vec![+ None,+ None,+ None,+ None,+ Some(9),+ Some(10),+ Some(10),+ Some(11),+ ])),+ Arc::new(Int32Array::from(vec![+ Some(9),+ Some(10),+ Some(11),+ None,+ None,+ Some(9),+ None,+ None,+ ])),+ ],+ )?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = result+ .as_any()+ .downcast_ref::<BooleanArray>()+ .expect("dynamic filter should evaluate to BooleanArray");+ assert_eq!(+ result,+ &BooleanArray::from(+ vec![false, true, true, false, false, false, true, true,]+ )+ );++ Ok(())+ }++ #[test]+ fn partitioned_range_dynamic_filter_preserves_signed_zero_routing() -> Result<()> {+ let probe_schema = Arc::new(Schema::new(vec![Field::new(+ "probe_key",+ DataType::Float64,+ false,+ )]));+ let on_right: Vec<PhysicalExprRef> = vec![Arc::new(Column::new("probe_key", 0))];+ let mut acc = make_accumulator_for_test(+ AccumulatedBuildData::Partitioned {+ partitions: vec![PartitionStatus::Pending; 2],+ completed_partitions: 0,+ },+ on_right,+ );+ acc.probe_schema = Arc::clone(&probe_schema);+ acc.probe_range_partitioning = Some(RangePartitioning::try_new(+ [PhysicalSortExpr::new(+ Arc::clone(&acc.on_right[0]),+ SortOptions::default(),+ )]+ .into(),+ vec![SplitPoint::new(vec![ScalarValue::Float64(Some(0.0))])],+ )?);++ acc.build_filter(FinalizeInput::Partitioned(vec![+ PartitionStatus::CanceledUnknown,+ reported(PushdownStrategy::Empty, no_bounds()),+ ]))?;++ let expr = current_expr(&acc);+ let batch = RecordBatch::try_new(+ probe_schema,+ vec![Arc::new(Float64Array::from(vec![-0.0, 0.0]))],+ )?;+ let result = expr.evaluate(&batch)?.into_array(batch.num_rows())?;+ let result = result+ .as_any()+ .downcast_ref::<BooleanArray>()+ .expect("dynamic filter should evaluate to BooleanArray");+ assert_eq!(result, &BooleanArray::from(vec![true, false]));++ Ok(())+ }+// Regression guard for the build-report lifecycle fix: on `Drop`, a stream// in `BuildReportState::ReportScheduled` still calls `report_canceled_partition`// because it cannot tell whether the coordinator has already observed the@@ -1153,6 +1417,7 @@ mod tests {on_right,repartition_random_state: SeededRandomState::with_seed(1),probe_schema,+ probe_range_partitioning: None,null_equality,null_aware,}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 51e350b2f0..8e7bec5dee 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -20,7 +20,7 @@//! maintaining the order of the input rows in the output.use std::cmp::Ordering;-use std::fmt::{Debug, Formatter};+use std::fmt::{Debug, Display, Formatter};use std::pin::Pin;use std::sync::Arc;use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};@@ -47,9 +47,9 @@ use crate::{check_if_same_properties,};-use arrow::array::{Array, PrimitiveArray, RecordBatch, RecordBatchOptions};+use arrow::array::{Array, PrimitiveArray, RecordBatch, RecordBatchOptions, UInt64Array};use arrow::compute::take_arrays;-use arrow::datatypes::{SchemaRef, UInt32Type};+use arrow::datatypes::{DataType, Schema, SchemaRef, UInt32Type};use arrow_schema::SortOptions;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;@@ -58,13 +58,22 @@ use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, transposuse datafusion_common::{ColumnStatistics, DataFusionError, HashMap, ScalarValue, SplitPoint,assert_or_internal_err, internal_datafusion_err, internal_err,+ validate_range_split_points,};use datafusion_common::{Result, not_impl_err};use datafusion_common_runtime::SpawnedTask;use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryConsumer;+use datafusion_expr::ColumnarValue;use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr, RangePartitioning};+use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;use datafusion_physical_expr_common::sort_expr::LexOrdering;+#[cfg(feature = "proto")]+use datafusion_physical_expr_common::sort_expr::{+ sort_exprs_try_from_proto, sort_exprs_try_to_proto,+};+#[cfg(feature = "proto")]+use datafusion_proto_models::protobuf;use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,@@ -640,6 +649,219 @@ enum BatchPartitionerState {/// executions and runs.pub const REPARTITION_RANDOM_STATE: SeededRandomState = SeededRandomState::with_seed(0);+/// Physical expression that returns the Range partition for each input row.+///+/// This uses the same routing function as [`BatchPartitioner`], so dynamic+/// filtering and repartitioning agree for every [`ScalarValue`] comparison.+#[derive(Debug, Hash, PartialEq, Eq)]+pub struct RangeExpr {+ on_columns: Vec<PhysicalExprRef>,+ split_points: Vec<SplitPoint>,+ sort_options: Vec<SortOptions>,+}++impl RangeExpr {+ /// Creates a Range expression for `on_columns` using the supplied routing+ /// metadata.+ pub fn try_new(+ on_columns: Vec<PhysicalExprRef>,+ range_partitioning: &RangePartitioning,+ ) -> Result<Self> {+ let sort_options = range_partitioning+ .ordering()+ .iter()+ .map(|expr| expr.options)+ .collect();+ Self::try_new_parts(+ on_columns,+ range_partitioning.split_points().to_vec(),+ sort_options,+ )+ }++ fn try_new_parts(+ on_columns: Vec<PhysicalExprRef>,+ split_points: Vec<SplitPoint>,+ sort_options: Vec<SortOptions>,+ ) -> Result<Self> {+ assert_or_internal_err!(!on_columns.is_empty(), "RangeExpr requires a key");+ assert_or_internal_err!(+ on_columns.len() == sort_options.len(),+ "RangeExpr key count must match sort options"+ );+ validate_range_split_points(&split_points, &sort_options)?;+ Ok(Self {+ on_columns,+ split_points,+ sort_options,+ })+ }++ /// Get the columns used to compute Range partition IDs.+ pub fn on_columns(&self) -> &[PhysicalExprRef] {+ &self.on_columns+ }++ /// Returns the Range split points used for routing.+ pub fn split_points(&self) -> &[SplitPoint] {+ &self.split_points+ }++ /// Returns the per-key sort options used for routing.+ pub fn sort_options(&self) -> &[SortOptions] {+ &self.sort_options+ }+}++impl Display for RangeExpr {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "range_partition")+ }+}++impl PhysicalExpr for RangeExpr {+ fn children(&self) -> Vec<&PhysicalExprRef> {+ self.on_columns.iter().collect()+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<PhysicalExprRef>,+ ) -> Result<PhysicalExprRef> {+ assert_or_internal_err!(+ children.len() == self.on_columns.len(),+ "RangeExpr expected {} children, got {}",+ self.on_columns.len(),+ children.len()+ );+ Ok(Arc::new(Self::try_new_parts(+ children,+ self.split_points.clone(),+ self.sort_options.clone(),+ )?))+ }++ fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {+ Ok(DataType::UInt64)+ }++ fn nullable(&self, _input_schema: &Schema) -> Result<bool> {+ Ok(false)+ }++ fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {+ let arrays = evaluate_expressions_to_arrays(self.on_columns.iter(), batch)?;+ let mut row_key_buffer = Vec::with_capacity(arrays.len());+ let mut partition_ids = Vec::with_capacity(batch.num_rows());+ for row_idx in 0..batch.num_rows() {+ extract_row_at_idx_to_buf(&arrays, row_idx, &mut row_key_buffer)?;+ partition_ids.push(range_partition_id(+ &row_key_buffer,+ &self.split_points,+ &self.sort_options,+ )? as u64);+ }+ Ok(ColumnarValue::Array(Arc::new(UInt64Array::from(+ partition_ids,+ ))))+ }++ fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ write!(f, "range_partition")+ }++ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<protobuf::PhysicalExprNode>> {+ // Encode the raw ordered children: rebuilding a `LexOrdering` would+ // deduplicate equivalent children after dynamic-filter remapping.+ let sort_exprs = self+ .on_columns+ .iter()+ .zip(&self.sort_options)+ .map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options))+ .collect::<Vec<_>>();+ let sort_expr = sort_exprs_try_to_proto(&sort_exprs, ctx)?;+ let split_point = self+ .split_points+ .iter()+ .map(|split_point| {+ let value = split_point+ .values()+ .iter()+ .map(|value| value.try_into().map_err(Into::into))+ .collect::<Result<Vec<_>>>()?;+ Ok(protobuf::PhysicalRangeSplitPoint { value })+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(protobuf::physical_expr_node::ExprType::RangeExpr(+ protobuf::PhysicalRangeExprNode {+ sort_expr,+ split_point,+ },+ )),+ }))+ }+}++#[cfg(feature = "proto")]+impl RangeExpr {+ /// Reconstructs a [`RangeExpr`] from its protobuf representation.+ pub fn try_from_proto(+ node: &protobuf::PhysicalExprNode,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<PhysicalExprRef> {+ // Decode the raw ordered children for the same reason as `try_to_proto`.+ let range_expr = match &node.expr_type {+ Some(protobuf::physical_expr_node::ExprType::RangeExpr(expr)) => expr,+ _ => return internal_err!("PhysicalExprNode is not a RangeExpr"),+ };+ let sort_exprs = sort_exprs_try_from_proto(&range_expr.sort_expr, ctx)?;+ let (on_columns, sort_options) = sort_exprs+ .into_iter()+ .map(|sort_expr| (sort_expr.expr, sort_expr.options))+ .unzip();+ let split_points = range_expr+ .split_point+ .iter()+ .map(|split_point| {+ let values = split_point+ .value+ .iter()+ .map(|value| ScalarValue::try_from(value).map_err(Into::into))+ .collect::<Result<Vec<_>>>()?;+ Ok(SplitPoint::new(values))+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(Arc::new(Self::try_new_parts(+ on_columns,+ split_points,+ sort_options,+ )?))+ }+}++fn range_partition_id(+ row_key: &[ScalarValue],+ split_points: &[SplitPoint],+ sort_options: &[SortOptions],+) -> Result<usize> {+ let mut low = 0;+ let mut high = split_points.len();+ while low < high {+ let mid = low + (high - low) / 2;+ match compare_rows(row_key, split_points[mid].values(), sort_options)? {+ Ordering::Less => high = mid,+ Ordering::Equal | Ordering::Greater => low = mid + 1,+ }+ }+ Ok(low)+}+/// Computes `value % divisor` without division in the hot loop when `divisor`/// is fixed for many values.///@@ -974,22 +1196,9 @@ impl BatchPartitioner {// Note that `extract_row_at_idx_to_buf` clears the `row_key_buffer` on each invocation, creating a new row key for comparison for each rowextract_row_at_idx_to_buf(arrays, row_idx, row_key_buffer)?;- let mut low = 0;- let mut high = split_points.len();- while low < high {- let mid = low + (high - low) / 2;- let comparison = compare_rows(- row_key_buffer,- split_points[mid].values(),- sort_options,- )?;- match comparison {- Ordering::Less => high = mid,- Ordering::Equal | Ordering::Greater => low = mid + 1,- }- }-- indices[low].push(row_idx as u32)+ let partition =+ range_partition_id(row_key_buffer, split_points, sort_options)?;+ indices[partition].push(row_idx as u32)}Ok(())@@ -1726,9 +1935,7 @@ impl ExecutionPlan for RepartitionExec {fn try_to_proto(&self,ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,- ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {- use datafusion_proto_models::protobuf;-+ ) -> Result<Option<protobuf::PhysicalPlanNode>> {let input = ctx.encode_child(self.input())?;let partitioning = self.partitioning().try_to_proto(&ctx.expr_ctx())?;@@ -1751,11 +1958,9 @@ impl ExecutionPlan for RepartitionExec {impl RepartitionExec {/// Reconstruct a [`RepartitionExec`] from its protobuf representation.pub fn try_from_proto(- node: &datafusion_proto_models::protobuf::PhysicalPlanNode,+ node: &protobuf::PhysicalPlanNode,ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {- use datafusion_proto_models::protobuf;-let repart = crate::expect_plan_variant!(node,protobuf::physical_plan_node::PhysicalPlanType::Repartition,@@ -2288,6 +2493,47 @@ mod tests {}}+ #[test]+ fn range_expr_preserves_duplicate_remapped_children() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::UInt32, false),+ Field::new("b", DataType::UInt32, false),+ ]));+ let sort_options = [SortOptions::new(false, false), SortOptions::new(true, true)];+ let split_points = vec![SplitPoint::new(vec![+ ScalarValue::UInt32(Some(10)),+ ScalarValue::UInt32(Some(20)),+ ])];+ let range_partitioning = RangePartitioning::try_new(+ [+ PhysicalSortExpr::new(col("a", &schema)?, sort_options[0]),+ PhysicalSortExpr::new(col("b", &schema)?, sort_options[1]),+ ]+ .into(),+ split_points.clone(),+ )?;+ let expr = Arc::new(RangeExpr::try_new(+ vec![col("a", &schema)?, col("b", &schema)?],+ &range_partitioning,+ )?);+ let remapped = col("a", &schema)?;+ let rewritten =+ expr.with_new_children(vec![Arc::clone(&remapped), Arc::clone(&remapped)])?;++ let rewritten = rewritten+ .downcast_ref::<RangeExpr>()+ .expect("rewritten expression should remain a RangeExpr");+ assert_eq!(rewritten.on_columns().len(), 2);+ assert!(Arc::ptr_eq(+ &rewritten.on_columns()[0],+ &rewritten.on_columns()[1]+ ));+ assert_eq!(rewritten.sort_options(), sort_options);+ assert_eq!(rewritten.split_points(), split_points);++ Ok(())+ }+#[test]fn strength_reduced_u64_remainder_matches_modulo() {let divisors = [diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 99b4ef6272..43a90264c2 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1050,6 +1050,7 @@ message PhysicalExprNode {PhysicalHigherOrderUdfNode higher_order_udf = 24;PhysicalLambdaExprNode lambda = 25;PhysicalLambdaVariableExprNode lambda_variable = 26;+ PhysicalRangeExprNode range_expr = 27;}}@@ -1202,6 +1203,11 @@ message PhysicalHashExprNode {string description = 6;}+message PhysicalRangeExprNode {+ repeated PhysicalSortExprNode sort_expr = 1;+ repeated PhysicalRangeSplitPoint split_point = 2;+}+message FilterExecNode {PhysicalPlanNode input = 1;PhysicalExprNode expr = 2;diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex 61b1ea3ff1..908f9752b7 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -18489,6 +18489,9 @@ impl serde::Serialize for PhysicalExprNode {physical_expr_node::ExprType::LambdaVariable(v) => {struct_ser.serialize_field("lambdaVariable", v)?;}+ physical_expr_node::ExprType::RangeExpr(v) => {+ struct_ser.serialize_field("rangeExpr", v)?;+ }}}struct_ser.end()@@ -18544,6 +18547,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"lambda","lambda_variable","lambdaVariable",+ "range_expr",+ "rangeExpr",];#[allow(clippy::enum_variant_names)]@@ -18573,6 +18578,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {HigherOrderUdf,Lambda,LambdaVariable,+ RangeExpr,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -18619,6 +18625,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"higherOrderUdf" | "higher_order_udf" => Ok(GeneratedField::HigherOrderUdf),"lambda" => Ok(GeneratedField::Lambda),"lambdaVariable" | "lambda_variable" => Ok(GeneratedField::LambdaVariable),+ "rangeExpr" | "range_expr" => Ok(GeneratedField::RangeExpr),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -18816,6 +18823,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {return Err(serde::de::Error::duplicate_field("lambdaVariable"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::LambdaVariable)+;+ }+ GeneratedField::RangeExpr => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("rangeExpr"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::RangeExpr);}}@@ -20876,6 +20890,116 @@ impl<'de> serde::Deserialize<'de> for PhysicalPlanNode {deserializer.deserialize_struct("datafusion.PhysicalPlanNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalRangeExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.sort_expr.is_empty() {+ len += 1;+ }+ if !self.split_point.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalRangeExprNode", len)?;+ if !self.sort_expr.is_empty() {+ struct_ser.serialize_field("sortExpr", &self.sort_expr)?;+ }+ if !self.split_point.is_empty() {+ struct_ser.serialize_field("splitPoint", &self.split_point)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalRangeExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "sort_expr",+ "sortExpr",+ "split_point",+ "splitPoint",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ SortExpr,+ SplitPoint,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "sortExpr" | "sort_expr" => Ok(GeneratedField::SortExpr),+ "splitPoint" | "split_point" => Ok(GeneratedField::SplitPoint),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalRangeExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalRangeExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalRangeExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut sort_expr__ = None;+ let mut split_point__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::SortExpr => {+ if sort_expr__.is_some() {+ return Err(serde::de::Error::duplicate_field("sortExpr"));+ }+ sort_expr__ = Some(map_.next_value()?);+ }+ GeneratedField::SplitPoint => {+ if split_point__.is_some() {+ return Err(serde::de::Error::duplicate_field("splitPoint"));+ }+ split_point__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(PhysicalRangeExprNode {+ sort_expr: sort_expr__.unwrap_or_default(),+ split_point: split_point__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalRangeExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalRangePartitioning {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 233b5fee1b..ba00577ab9 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -1557,7 +1557,7 @@ pub struct PhysicalExprNode {pub expr_id: ::core::option::Option<u64>,#[prost(oneof = "physical_expr_node::ExprType",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27")]pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,}@@ -1620,6 +1620,8 @@ pub mod physical_expr_node {Lambda(::prost::alloc::boxed::Box<super::PhysicalLambdaExprNode>),#[prost(message, tag = "26")]LambdaVariable(super::PhysicalLambdaVariableExprNode),+ #[prost(message, tag = "27")]+ RangeExpr(super::PhysicalRangeExprNode),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -1861,6 +1863,13 @@ pub struct PhysicalHashExprNode {pub description: ::prost::alloc::string::String,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalRangeExprNode {+ #[prost(message, repeated, tag = "1")]+ pub sort_expr: ::prost::alloc::vec::Vec<PhysicalSortExprNode>,+ #[prost(message, repeated, tag = "2")]+ pub split_point: ::prost::alloc::vec::Vec<PhysicalRangeSplitPoint>,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct FilterExecNode {#[prost(message, optional, boxed, tag = "1")]pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex b88c3cf28f..bb1cb26108 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -46,6 +46,7 @@ use datafusion_physical_plan::expressions::{};use datafusion_physical_plan::joins::HashExpr;use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;+use datafusion_physical_plan::repartition::RangeExpr;use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use datafusion_proto_common::common::proto_error;@@ -354,6 +355,7 @@ pub fn parse_physical_expr_with_converter(}ExprType::LikeExpr(_) => LikeExpr::try_from_proto(proto, &decode_ctx)?,ExprType::HashExpr(_) => HashExpr::try_from_proto(proto, &decode_ctx)?,+ ExprType::RangeExpr(_) => RangeExpr::try_from_proto(proto, &decode_ctx)?,ExprType::ScalarSubquery(_) => {let results = ctx.scalar_subquery_results().ok_or_else(|| {proto_error(diff --git a/datafusion/proto/tests/cases/plans/exprs.rs b/datafusion/proto/tests/cases/plans/exprs.rsindex ed9745a4b1..518b4a62ce 100644--- a/datafusion/proto/tests/cases/plans/exprs.rs+++ b/datafusion/proto/tests/cases/plans/exprs.rs@@ -18,18 +18,22 @@//! Physical expressions embedded in plans, including the binary//! expression linearization.-use super::roundtrip_test;+use super::{roundtrip_test, roundtrip_test_and_return};use arrow::datatypes::Fields;+use datafusion::arrow::compute::SortOptions;use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};use datafusion::logical_expr::Operator;use datafusion::physical_expr::expressions::Literal;use datafusion::physical_plan::empty::EmptyExec;use datafusion::physical_plan::expressions::{- BinaryExpr, Column, binary, col, like, lit,+ BinaryExpr, Column, PhysicalSortExpr, binary, col, like, lit,};use datafusion::physical_plan::filter::FilterExec;use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};-use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr};+use datafusion::physical_plan::repartition::RangeExpr;+use datafusion::physical_plan::{+ ExecutionPlan, PhysicalExpr, RangePartitioning, SplitPoint,+};use datafusion::prelude::SessionContext;use datafusion::scalar::ScalarValue;use datafusion_common::Result;@@ -174,6 +178,58 @@ fn roundtrip_hash_expr() -> Result<()> {roundtrip_test(filter)}+#[test]+fn roundtrip_range_expr() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Float64, false),+ Field::new("b", DataType::Float64, false),+ ]));+ let options = [SortOptions::new(true, true), SortOptions::new(false, false)];+ let range_partitioning = RangePartitioning::try_new(+ [+ PhysicalSortExpr::new(col("a", &schema)?, options[0]),+ PhysicalSortExpr::new(col("b", &schema)?, options[1]),+ ]+ .into(),+ vec![SplitPoint::new(vec![+ ScalarValue::Float64(Some(0.0)),+ ScalarValue::Float64(Some(1.0)),+ ])],+ )?;+ let range_expr: Arc<dyn PhysicalExpr> = Arc::new(RangeExpr::try_new(+ // Expression remapping may produce duplicate children. Preserve both+ // so their sort options stay aligned with the split-point values.+ vec![col("a", &schema)?, col("a", &schema)?],+ &range_partitioning,+ )?);+ let filter_expr = binary(range_expr, Operator::Eq, lit(0u64), &schema)?;+ let plan = Arc::new(FilterExec::try_new(+ filter_expr,+ Arc::new(EmptyExec::new(Arc::clone(&schema))),+ )?);++ let ctx = SessionContext::new();+ let result = roundtrip_test_and_return(+ plan,+ &ctx,+ &DefaultPhysicalExtensionCodec {},+ &DefaultPhysicalProtoConverter {},+ )?;+ let filter = result.downcast_ref::<FilterExec>().unwrap();+ let binary = filter.predicate().downcast_ref::<BinaryExpr>().unwrap();+ let range_expr = binary.left().downcast_ref::<RangeExpr>().unwrap();+ assert_eq!(range_expr.split_points(), range_partitioning.split_points());+ assert_eq!(range_expr.sort_options(), &options);+ let children = range_expr.on_columns();+ assert_eq!(children.len(), 2);+ for child in children {+ let column = child.downcast_ref::<Column>().unwrap();+ assert_eq!((column.name(), column.index()), ("a", 0));+ }++ Ok(())+}+#[test]fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {let data_type = DataType::Struct(Fields::from(vec![Field::new(diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex 4141e00014..3cde3939f0 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use std::fs::{create_dir_all, remove_dir_all, write};+use std::fs::{File, create_dir_all, remove_dir_all};use std::path::Path;use std::sync::Arc;@@ -25,11 +25,12 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion::catalog::streaming::StreamingTable;use datafusion::common::{ScalarValue, SplitPoint};-use datafusion::datasource::file_format::csv::CsvFormat;+use datafusion::datasource::file_format::parquet::ParquetFormat;use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};use datafusion::logical_expr::{Partitioning, RangePartitioning, col};+use datafusion::parquet::arrow::ArrowWriter;use datafusion::physical_expr::{Partitioning as PhysicalPartitioning, PhysicalSortExpr,RangePartitioning as PhysicalRangePartitioning, expressions::col as physical_col,@@ -45,6 +46,30 @@ use datafusion::prelude::SessionContext;/// Registers a simple range-partitioned listing table for testing before/// declaring such tables is supported via SQL.pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {+ const RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [+ &[(1, 1, 10), (5, 2, 50)],+ &[(10, 1, 100), (15, 2, 150)],+ &[(20, 1, 200), (25, 2, 250)],+ &[(30, 1, 300), (35, 2, 350)],+ ];+ const SHIFTED_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [+ &[(1, 1, 10), (5, 2, 50), (10, 1, 100)],+ &[(15, 2, 150)],+ &[(20, 1, 200), (25, 2, 250)],+ &[(30, 1, 300), (35, 2, 350)],+ ];+ const NARROW_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 3] = [+ &[(1, 1, 10), (5, 2, 50)],+ &[(10, 1, 100), (15, 2, 150)],+ &[(20, 1, 200), (25, 2, 250), (30, 1, 300), (35, 2, 350)],+ ];+ const SPARSE_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [+ &[(5, 2, 50), (8, 3, 80)],+ &[(10, 1, 100)],+ &[(20, 1, 200)],+ &[(30, 1, 300), (40, 4, 400)],+ ];+let schema = Arc::new(Schema::new(vec![Field::new("range_key", DataType::Int32, false),Field::new("non_range_key", DataType::Int32, false),@@ -65,18 +90,13 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {let range_table_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("test_files/scratch_range_partitioning/range_partitioned");- register_csv_listing_table(+ register_parquet_listing_table(ctx,"range_partitioned",&range_table_dir,Arc::clone(&schema),- [- "1,1,10\n5,2,50\n",- "10,1,100\n15,2,150\n",- "20,1,200\n25,2,250\n",- "30,1,300\n35,2,350\n",- ],- Some(output_partitioning),+ RANGE_PARTITIONS,+ output_partitioning,);register_unbounded_range_stream_table(@@ -84,24 +104,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {"unbounded_range_like",Arc::clone(&schema),[10, 20, 30],- [- vec![(1, 1, 10), (5, 2, 50)],- vec![(10, 1, 100), (15, 2, 150)],- vec![(20, 1, 200), (25, 2, 250)],- vec![(30, 1, 300), (35, 2, 350)],- ],+ RANGE_PARTITIONS.map(|rows| rows.to_vec()),);register_unbounded_range_stream_table(ctx,"unbounded_range_like_shifted",Arc::clone(&schema),[15, 20, 30],- [- vec![(1, 1, 10), (5, 2, 50), (10, 1, 100)],- vec![(15, 2, 150)],- vec![(20, 1, 200), (25, 2, 250)],- vec![(30, 1, 300), (35, 2, 350)],- ],+ SHIFTED_RANGE_PARTITIONS.map(|rows| rows.to_vec()),);let shifted_output_partitioning = Partitioning::Range(@@ -116,19 +126,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {.expect("range partitioning should be valid"),);- register_csv_listing_table(+ register_parquet_listing_table(ctx,"range_partitioned_shifted",Path::new(env!("CARGO_MANIFEST_DIR")).join("test_files/scratch_range_partitioning/range_partitioned_shifted"),Arc::clone(&schema),- [- "1,1,10\n5,2,50\n10,1,100\n",- "15,2,150\n",- "20,1,200\n25,2,250\n",- "30,1,300\n35,2,350\n",- ],- Some(shifted_output_partitioning),+ SHIFTED_RANGE_PARTITIONS,+ shifted_output_partitioning,);// Same rows as `range_partitioned` but split into only three range@@ -145,18 +150,14 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {.expect("range partitioning should be valid"),);- register_csv_listing_table(+ register_parquet_listing_table(ctx,"range_partitioned_narrow",Path::new(env!("CARGO_MANIFEST_DIR")).join("test_files/scratch_range_partitioning/range_partitioned_narrow"),Arc::clone(&schema),- [- "1,1,10\n5,2,50\n",- "10,1,100\n15,2,150\n",- "20,1,200\n25,2,250\n30,1,300\n35,2,350\n",- ],- Some(narrow_output_partitioning),+ NARROW_RANGE_PARTITIONS,+ narrow_output_partitioning,);let sparse_output_partitioning = Partitioning::Range(@@ -171,29 +172,24 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {.expect("range partitioning should be valid"),);- register_csv_listing_table(+ register_parquet_listing_table(ctx,"range_partitioned_sparse",Path::new(env!("CARGO_MANIFEST_DIR")).join("test_files/scratch_range_partitioning/range_partitioned_sparse"),- schema,- [- "5,2,50\n8,3,80\n",- "10,1,100\n",- "20,1,200\n",- "30,1,300\n40,4,400\n",- ],- Some(sparse_output_partitioning),+ Arc::clone(&schema),+ SPARSE_RANGE_PARTITIONS,+ sparse_output_partitioning,);}-fn register_csv_listing_table(+fn register_parquet_listing_table(ctx: &SessionContext,name: &str,table_dir: impl AsRef<Path>,- schema: Arc<Schema>,- partitions: impl IntoIterator<Item = &'static str>,- output_partitioning: Option<Partitioning>,+ schema: SchemaRef,+ partitions: impl IntoIterator<Item = &'static [(i32, i32, i32)]>,+ output_partitioning: Partitioning,) {let table_dir = table_dir.as_ref();if table_dir.exists() {@@ -201,8 +197,17 @@ fn register_csv_listing_table(}create_dir_all(table_dir).expect("test table dir should be created");for (idx, rows) in partitions.into_iter().enumerate() {- write(table_dir.join(format!("part-{idx}.csv")), rows)- .expect("test table csv partition should be written");+ let batch = range_batch(Arc::clone(&schema), rows);+ let file = File::create(table_dir.join(format!("part-{idx}.parquet")))+ .expect("test table parquet partition should be created");+ let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), None)+ .expect("test table parquet writer should be created");+ writer+ .write(&batch)+ .expect("test table parquet partition should be written");+ writer+ .close()+ .expect("test table parquet writer should close");}let table_path = format!(@@ -213,9 +218,8 @@ fn register_csv_listing_table();let table_url =ListingTableUrl::parse(&table_path).expect("test table url should parse");- let options =- ListingOptions::new(Arc::new(CsvFormat::default().with_has_header(false)))- .with_output_partitioning(output_partitioning);+ let options = ListingOptions::new(Arc::new(ParquetFormat::default()))+ .with_output_partitioning(Some(output_partitioning));let config = ListingTableConfig::new(table_url).with_listing_options(options).with_schema(schema);@@ -269,20 +273,22 @@ fn range_stream_partition(schema: SchemaRef,rows: &[(i32, i32, i32)],) -> Arc<dyn PartitionStream> {- let range_key: Vec<i32> = rows.iter().map(|(range_key, _, _)| *range_key).collect();- let non_range_key: Vec<i32> = rows- .iter()- .map(|(_, non_range_key, _)| *non_range_key)- .collect();- let value: Vec<i32> = rows.iter().map(|(_, _, value)| *value).collect();- let batch = RecordBatch::try_new(+ Arc::new(TestPartitionStream::new_with_batches(vec![range_batch(+ schema, rows,+ )]))+}++fn range_batch(schema: SchemaRef, rows: &[(i32, i32, i32)]) -> RecordBatch {+ RecordBatch::try_new(schema,vec![- Arc::new(Int32Array::from(range_key)) as ArrayRef,- Arc::new(Int32Array::from(non_range_key)) as ArrayRef,- Arc::new(Int32Array::from(value)) as ArrayRef,+ Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.0)))+ as ArrayRef,+ Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.1)))+ as ArrayRef,+ Arc::new(Int32Array::from_iter_values(rows.iter().map(|row| row.2)))+ as ArrayRef,],)- .expect("range stream batch should be valid");- Arc::new(TestPartitionStream::new_with_batches(vec![batch]))+ .expect("range batch should be valid")}diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 9701c41377..326856a352 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -16,7 +16,7 @@# under the License.# The sqllogictest harness registers range_partitioned(range_key, non_range_key, value)-# as a CSV ListingTable with four declared range-partitioned file groups:+# as a Parquet ListingTable with four declared range-partitioned file groups:## partition 0: range_key in [..., 10), rows (1, 1, 10), (5, 2, 50)# partition 1: range_key in [10, 20), rows (10, 1, 100), (15, 2, 150)@@ -26,6 +26,21 @@statement okset datafusion.explain.physical_plan_only = true;+statement ok+set datafusion.execution.collect_statistics = false;++statement ok+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = false;++statement ok+set datafusion.optimizer.enable_topk_dynamic_filter_pushdown = false;++statement ok+set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = false;++statement ok+set datafusion.execution.parquet.pushdown_filters = false;+########### TEST 1: Aggregate on Range Partition Column# With subset threshold met and preserve-file disabled, Range([range_key])@@ -43,7 +58,7 @@ EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;----physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key ORDER BY range_key;@@ -77,7 +92,7 @@ physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)]02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=403)----AggregateExec: mode=Partial, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquetquery IISELECT non_range_key, SUM(value) FROM range_partitioned GROUP BY non_range_key ORDER BY non_range_key;@@ -103,7 +118,7 @@ EXPLAIN SELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP----physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key, non_range_key@1 as non_range_key], aggr=[sum(range_partitioned.value)]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT range_key, non_range_key, SUM(value) FROM range_partitioned GROUP BY range_key, non_range_key ORDER BY range_key, non_range_key;@@ -138,7 +153,7 @@ EXPLAIN SELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key;----physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetstatement okset datafusion.execution.target_partitions = 4;@@ -170,7 +185,7 @@ physical_plan02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=503)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]04)------RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetstatement okset datafusion.execution.target_partitions = 4;@@ -202,8 +217,8 @@ JOIN range_partitioned r ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -236,9 +251,9 @@ JOIN range_partitioned_shifted r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -269,9 +284,9 @@ JOIN range_partitioned r ON l.non_range_key = r.non_range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(non_range_key@0, non_range_key@0)], projection=[non_range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet04)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquetquery IIISELECT l.non_range_key, l.value, r.value@@ -326,9 +341,9 @@ ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet03)--FilterExec: value@1 <= 150-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -354,9 +369,9 @@ ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(range_key@0, range_key@0)]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet03)--FilterExec: value@1 <= 150, projection=[range_key@0]-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT l.range_key, l.value@@ -378,9 +393,9 @@ ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(range_key@0, range_key@0)]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet03)--FilterExec: value@1 <= 150, projection=[range_key@0]-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT l.range_key, l.value@@ -411,10 +426,10 @@ ON l.range_key = r.range_key AND l.non_range_key = r.non_range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5]02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=405)----FilterExec: value@2 <= 150-06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet# Range([range_key]) does not satisfy a join keyed on non_range_key.query TT@@ -425,9 +440,9 @@ LEFT JOIN range_partitioned r ON l.non_range_key = r.non_range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(non_range_key@1, non_range_key@0)], projection=[range_key@0, non_range_key@1, value@2, value@4]02)--RepartitionExec: partitioning=Hash([non_range_key@1], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquet########### TEST 11: Left-Side Range Hash Joins With Incompatible Range Layouts@@ -444,9 +459,9 @@ LEFT JOIN range_partitioned_shifted r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -473,9 +488,9 @@ LEFT JOIN range_partitioned_narrow r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Left, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=3-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=parquet########### TEST 12: LeftMark Subqueries Over Range Hash Joins@@ -492,9 +507,9 @@ WHERE l.non_range_key = 2 OR l.range_key IN (physical_plan01)FilterExec: non_range_key@1 = 2 OR mark@3, projection=[range_key@0, value@2]02)--HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(range_key@0, range_key@0)]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)----FilterExec: value@1 <= 150, projection=[range_key@0]-05)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT l.range_key, l.value@@ -543,9 +558,9 @@ JOIN range_partitioned r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 5), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -578,8 +593,8 @@ JOIN range_partitioned r ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -617,9 +632,9 @@ JOIN range_partitioned s ON r.range_key = s.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@2, range_key@0)], projection=[range_key@0, value@1, value@3, value@5]02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-05)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+05)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIIISELECT l.range_key, l.value, r.value, s.value@@ -662,10 +677,10 @@ physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, l_sum@1, r_sum@3]02)--ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value)@1 as l_sum]03)----AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet05)--ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value)@1 as r_sum]06)----AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-07)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+07)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIIWITH@@ -707,8 +722,8 @@ GROUP BY l.range_key;physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(l.value + r.value)]02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT l.range_key, SUM(l.value + r.value)@@ -741,8 +756,8 @@ RIGHT JOIN range_partitioned r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3]02)--FilterExec: value@1 <= 150-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[]+04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.value, r.range_key, r.value@@ -773,8 +788,8 @@ RIGHT SEMI JOIN range_partitioned r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(range_key@0, range_key@0)]02)--FilterExec: value@1 <= 150, projection=[range_key@0]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[]+04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT r.range_key, r.value@@ -801,8 +816,8 @@ RIGHT ANTI JOIN range_partitioned r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(range_key@0, range_key@0)]02)--FilterExec: value@1 <= 150, projection=[range_key@0]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[]+04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT r.range_key, r.value@@ -831,9 +846,9 @@ physical_plan01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=403)----FilterExec: value@1 <= 150-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=value@2 <= 150, pruning_predicate=value_null_count@1 != row_count@2 AND value_min@0 <= 150, required_guarantees=[]05)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-06)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+06)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IIISELECT l.value, r.range_key, r.value@@ -867,9 +882,9 @@ RIGHT JOIN range_partitioned r ON l.range_key = r.range_key AND l.non_range_keyphysical_plan01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5]02)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIIISELECT l.range_key, l.non_range_key, l.value, r.value@@ -905,9 +920,9 @@ RIGHT JOIN range_partitioned_narrow r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(range_key@0, range_key@0)], projection=[value@1, range_key@2, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=3-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20)], 3), file_type=parquetquery IIISELECT l.value, r.range_key, r.value@@ -940,9 +955,9 @@ physical_plan01)HashJoinExec: mode=Partitioned, join_type=Right, on=[(non_range_key@0, non_range_key@1)], projection=[value@1, range_key@2, value@4]02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=403)----FilterExec: range_key@0 < 10, projection=[non_range_key@1, value@2]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 < 10, pruning_predicate=range_key_null_count@1 != row_count@2 AND range_key_min@0 < 10, required_guarantees=[]05)--RepartitionExec: partitioning=Hash([non_range_key@1], 4), input_partitions=4-06)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+06)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.value, r.range_key, r.value@@ -974,9 +989,9 @@ WHERE r.non_range_key = 2 OR r.range_key IN (physical_plan01)FilterExec: non_range_key@1 = 2 OR mark@3, projection=[range_key@0, value@2]02)--HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(range_key@0, range_key@0)]-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)----FilterExec: value@1 <= 150, projection=[range_key@0]-05)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet# Matched rows have mark=true and are returned; unmatched rows have# mark=false and are only returned when non_range_key = 2.@@ -1029,9 +1044,9 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, sort_order_for_reorder=[range_key@0 ASC]05)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]-06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, sort_order_for_reorder=[range_key@0 ASC]query IIISELECT l.range_key, l.value, r.value@@ -1064,10 +1079,10 @@ physical_plan02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]04)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet06)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]07)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-08)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+08)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -1167,8 +1182,8 @@ FULL JOIN range_partitioned r ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -1199,9 +1214,9 @@ FULL JOIN range_partitioned_shifted r ON l.range_key = r.range_key;physical_plan01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet04)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+05)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IIISELECT l.range_key, l.value, r.value@@ -1234,8 +1249,8 @@ FULL JOIN range_partitioned_sparse r ON l.range_key = r.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)], projection=[range_key@0, range_key@2, value@1, value@3]-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIIISELECT l.range_key, r.range_key, l.value, r.value@@ -1276,8 +1291,8 @@ SELECT range_key, value FROM range_partitioned;----physical_plan01)InterleaveExec-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, value FROM range_partitioned@@ -1325,7 +1340,7 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortExec: expr=[range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value) FROM range_partitioned ORDER BY range_key;@@ -1354,7 +1369,7 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING]02)--WindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]03)----SortExec: expr=[range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM range_partitioned ORDER BY range_key;@@ -1383,7 +1398,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortExec: expr=[non_range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true]04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquetquery IIISELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER BY value) FROM range_partitioned ORDER BY non_range_key, value;@@ -1413,7 +1428,7 @@ physical_plan02)--WindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(range_partitioned.value) PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Int64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }]03)----SortExec: expr=[non_range_key@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true]04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquetquery IIISELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER BY value ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM range_partitioned ORDER BY non_range_key, value;@@ -1445,7 +1460,7 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortExec: expr=[range_key@0 ASC NULLS LAST, non_range_key@1 ASC NULLS LAST, value@2 ASC NULLS LAST], preserve_partitioning=[true]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER BY value) FROM range_partitioned ORDER BY range_key;@@ -1478,7 +1493,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortExec: expr=[range_key@0 ASC NULLS LAST, non_range_key@1 ASC NULLS LAST, value@2 ASC NULLS LAST], preserve_partitioning=[true]04)------RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER BY value) FROM range_partitioned ORDER BY range_key;@@ -1513,7 +1528,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[sum(range_partitioned.value) ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(range_partitioned.value) ORDER BY [range_partitioned.value ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortPreservingMergeExec: [value@1 ASC NULLS LAST]04)------SortExec: expr=[value@1 ASC NULLS LAST], preserve_partitioning=[true]-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) OVER (ORDER BY value) FROM range_partitioned ORDER BY range_key;@@ -1553,7 +1568,7 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rn]02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0], order=[value@1 DESC]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIISELECT * FROM (@@ -1595,7 +1610,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[non_range_key@0], order=[value@1 DESC]04)------RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=parquetquery IIISELECT * FROM (@@ -1630,7 +1645,7 @@ physical_plan01)ProjectionExec: expr=[range_key@0 as range_key, non_range_key@1 as non_range_key, value@2 as value, row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn]02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IIIISELECT * FROM (@@ -1676,7 +1691,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [range_partitioned.range_key, range_partitioned.non_range_key] ORDER BY [range_partitioned.value DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----PartitionedTopKExec: fn=row_number, fetch=1, partition=[range_key@0, non_range_key@1], order=[value@2 DESC]04)------RepartitionExec: partitioning=Hash([range_key@0, non_range_key@1], 4), input_partitions=4-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetstatement okreset datafusion.optimizer.subset_repartition_threshold;@@ -1706,9 +1721,9 @@ SELECT range_key, value FROM range_partitioned_shifted;----physical_plan01)UnionExec-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+04)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IISELECT range_key, value FROM range_partitioned@@ -1759,8 +1774,8 @@ SELECT range_key, value FROM range_partitioned_shifted;----physical_plan01)UnionExec-02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=parquetquery IISELECT range_key, value FROM range_partitioned@@ -1802,8 +1817,8 @@ EXPLAIN SELECT range_key, SUM(value) FROM (physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(value)]02)--InterleaveExec-03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquetquery IISELECT range_key, SUM(value) FROM (@@ -1821,5 +1836,84 @@ SELECT range_key, SUM(value) FROM (30 60035 700+##########+# TEST 48: Hash Join Dynamic Filter Pushdown on Compatible Range Inputs+# The Parquet-backed probe accepts the partition-routed dynamic filter. Matching+# Range split points keep build filter i aligned with probe partition i. The+# build has rows only in partitions 0 and 2, so the runtime filter must route+# with a four-way CASE rather than collapse to a single filter.+##########++statement ok+set datafusion.execution.target_partitions = 4;++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++statement ok+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = true;++statement ok+set datafusion.execution.parquet.pushdown_filters = true;++query TT+EXPLAIN SELECT b.range_key, b.value, p.value+FROM (+ SELECT range_key, value+ FROM range_partitioned+ WHERE range_key IN (5, 20)+) b+JOIN range_partitioned p ON b.range_key = p.range_key;+----+physical_plan+01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20, pruning_predicate=range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 5 AND 5 <= range_key_max@1 OR range_key_null_count@2 != row_count@3 AND range_key_min@0 <= 20 AND 20 <= range_key_max@1, required_guarantees=[range_key in (20, 5)]+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible++query TT+EXPLAIN ANALYZE SELECT b.range_key, b.value, p.value+FROM (+ SELECT range_key, value+ FROM range_partitioned+ WHERE range_key IN (5, 20)+) b+JOIN range_partitioned p ON b.range_key = p.range_key;+----+Plan with Metrics+01)HashJoinExec: mode=Partitioned<slt:ignore>metrics=[output_rows=2,<slt:ignore>]+02)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=range_key@0 = 5 OR range_key@0 = 20<slt:ignore>metrics=[output_rows=2,<slt:ignore>]+03)--DataSourceExec: <slt:ignore>file_type=parquet, predicate=DynamicFilter [ CASE range_partition WHEN 0 THEN range_key@0 >= 5 AND range_key@0 <= 5 AND range_key@0 IN (SET) ([5]) WHEN 1 THEN false WHEN 2 THEN range_key@0 >= 20 AND range_key@0 <= 20 AND range_key@0 IN (SET) ([20]) ELSE false END ]<slt:ignore>metrics=[output_rows=2,<slt:ignore>]++query III+SELECT b.range_key, b.value, p.value+FROM (+ SELECT range_key, value+ FROM range_partitioned+ WHERE range_key IN (5, 20)+) b+JOIN range_partitioned p ON b.range_key = p.range_key+ORDER BY b.range_key;+----+5 50 50+20 200 200++statement ok+reset datafusion.optimizer.preserve_file_partitions;++statement ok+reset datafusion.execution.collect_statistics;++statement ok+reset datafusion.optimizer.enable_join_dynamic_filter_pushdown;++statement ok+reset datafusion.optimizer.enable_topk_dynamic_filter_pushdown;++statement ok+reset datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown;++statement ok+reset datafusion.execution.parquet.pushdown_filters;+statement okreset datafusion.explain.physical_plan_only;