Apache DataFusion SQL Query Engine
Enforce co-partitioning for sort merge and symmetric hash joins (#23480)
Gene Bordegaray754e11335eda
diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rsindex e609877c2b..5bfecef1fb 100644--- a/datafusion/catalog/src/streaming.rs+++ b/datafusion/catalog/src/streaming.rs@@ -24,7 +24,10 @@ use async_trait::async_trait;use datafusion_common::{DFSchema, Result, plan_err};use datafusion_expr::{Expr, SortExpr, TableType};use datafusion_physical_expr::equivalence::project_ordering;-use datafusion_physical_expr::{LexOrdering, create_physical_sort_exprs};+use datafusion_physical_expr::projection::ProjectionMapping;+use datafusion_physical_expr::{+ EquivalenceProperties, LexOrdering, Partitioning, create_physical_sort_exprs,+};use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};use log::debug;@@ -38,6 +41,7 @@ pub struct StreamingTable {partitions: Vec<Arc<dyn PartitionStream>>,infinite: bool,sort_order: Vec<SortExpr>,+ output_partitioning: Option<Partitioning>,}impl StreamingTable {@@ -62,6 +66,7 @@ impl StreamingTable {partitions,infinite: false,sort_order: vec![],+ output_partitioning: None,})}@@ -76,6 +81,33 @@ impl StreamingTable {self.sort_order = sort_order;self}++ /// Declares the output partitioning of this streaming table.+ ///+ /// The partitioning expressions refer to the table schema before scan+ /// projection. If a scan projection removes a partitioning expression, the+ /// physical plan reports unknown partitioning.+ pub fn with_output_partitioning(mut self, output_partitioning: Partitioning) -> Self {+ self.output_partitioning = Some(output_partitioning);+ self+ }++ fn output_partitioning(+ &self,+ projection: Option<&Vec<usize>>,+ ) -> Result<Partitioning> {+ let Some(output_partitioning) = &self.output_partitioning else {+ return Ok(Partitioning::UnknownPartitioning(self.partitions.len()));+ };+ let Some(projection) = projection else {+ return Ok(output_partitioning.clone());+ };++ let projection_mapping =+ ProjectionMapping::from_indices(projection, &self.schema)?;+ let eq_properties = EquivalenceProperties::new(Arc::clone(&self.schema));+ Ok(output_partitioning.project(&projection_mapping, &eq_properties))+ }}#[async_trait]@@ -119,13 +151,16 @@ impl TableProvider for StreamingTable {vec![]};- Ok(Arc::new(StreamingTableExec::try_new(+ let exec = StreamingTableExec::try_new(Arc::clone(&self.schema),self.partitions.clone(),projection,LexOrdering::new(physical_sort),self.infinite,limit,- )?))+ )?+ .with_output_partitioning(self.output_partitioning(projection)?)?;++ Ok(Arc::new(exec))}}diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rsindex e5718f5b3d..3c426e2b09 100644--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs@@ -30,12 +30,13 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};use datafusion::prelude::{CsvReadOptions, SessionContext};use datafusion_common::config::ConfigOptions;-use datafusion_common::{JoinType, Result, ScalarValue};+use datafusion_common::{JoinType, NullEquality, Result, ScalarValue};use datafusion_physical_expr::expressions::{Literal, col};use datafusion_physical_expr::{Partitioning, RangePartitioning, SplitPoint};use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;+use datafusion_physical_plan::joins::{StreamJoinPartitionMode, SymmetricHashJoinExec};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::{ExecutionPlan, displayable};@@ -444,6 +445,77 @@ fn test_partitioned_hash_join_requires_co_partitioned_children() -> Result<()> {Ok(())}+#[test]+fn test_sort_merge_join_requires_co_partitioned_children() -> Result<()> {+ let schema = create_test_schema2();+ let join_on = vec![(col("a", &schema)?, col("a", &schema)?)];+ let ordering: LexOrdering = [sort_expr("a", &schema)].into();++ let compatible_join = sort_merge_join_exec(+ sort_exec_with_preserve_partitioning(+ ordering.clone(),+ range_partitioned_exec(&schema, "a", [10])?,+ ),+ sort_exec_with_preserve_partitioning(+ ordering.clone(),+ range_partitioned_exec(&schema, "a", [10])?,+ ),+ &join_on,+ &JoinType::Inner,+ );+ assert_sanity_check(&compatible_join, true);++ let incompatible_join = sort_merge_join_exec(+ sort_exec_with_preserve_partitioning(+ ordering.clone(),+ range_partitioned_exec(&schema, "a", [10])?,+ ),+ sort_exec_with_preserve_partitioning(+ ordering,+ range_partitioned_exec(&schema, "a", [20])?,+ ),+ &join_on,+ &JoinType::Inner,+ );+ assert_sanity_check(&incompatible_join, false);++ Ok(())+}++#[test]+fn test_symmetric_hash_join_requires_co_partitioned_children() -> Result<()> {+ let schema = create_test_schema2();+ let join_on = vec![(col("a", &schema)?, col("a", &schema)?)];++ let compatible_join = Arc::new(SymmetricHashJoinExec::try_new(+ range_partitioned_exec(&schema, "a", [10])?,+ range_partitioned_exec(&schema, "a", [10])?,+ join_on.clone(),+ None,+ &JoinType::Inner,+ NullEquality::NullEqualsNothing,+ None,+ None,+ StreamJoinPartitionMode::Partitioned,+ )?) as Arc<dyn ExecutionPlan>;+ assert_sanity_check(&compatible_join, true);++ let incompatible_join = Arc::new(SymmetricHashJoinExec::try_new(+ range_partitioned_exec(&schema, "a", [10])?,+ range_partitioned_exec(&schema, "a", [20])?,+ join_on,+ None,+ &JoinType::Inner,+ NullEquality::NullEqualsNothing,+ None,+ None,+ StreamJoinPartitionMode::Partitioned,+ )?) as Arc<dyn ExecutionPlan>;+ assert_sanity_check(&incompatible_join, false);++ Ok(())+}+#[tokio::test]/// Tests that plan is valid when the sort requirements are satisfied.async fn test_bounded_window_agg_sort_requirement() -> Result<()> {diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex 46c37696ec..5d3621c492 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -41,7 +41,8 @@ use crate::spill::spill_manager::SpillManager;use crate::statistics::{ChildStats, StatisticsArgs};use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- PlanProperties, SendableRecordBatchStream, Statistics, check_if_same_properties,+ InputDistributionRequirements, PlanProperties, SendableRecordBatchStream, Statistics,+ check_if_same_properties,};use arrow::compute::SortOptions;@@ -413,16 +414,17 @@ impl ExecutionPlan for SortMergeJoinExec {self.input_distribution_requirements().into_per_child()}- fn input_distribution_requirements(&self) -> crate::InputDistributionRequirements {+ fn input_distribution_requirements(&self) -> InputDistributionRequirements {let (left_expr, right_expr) = self.on.iter().map(|(l, r)| (Arc::clone(l), Arc::clone(r))).unzip();- crate::InputDistributionRequirements::new(vec![+ InputDistributionRequirements::co_partitioned(vec![Distribution::KeyPartitioned(left_expr),Distribution::KeyPartitioned(right_expr),])+ .allow_range_satisfaction_for_key_partitioning()}fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex f33d9b1d07..0aaf8ac608 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -53,7 +53,8 @@ use crate::projection::{use crate::stream::EmptyRecordBatchStream;use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- PlanProperties, RecordBatchStream, SendableRecordBatchStream,+ InputDistributionRequirements, PlanProperties, RecordBatchStream,+ SendableRecordBatchStream,joins::StreamJoinPartitionMode,metrics::{ExecutionPlanMetricsSet, MetricsSet},};@@ -415,23 +416,27 @@ impl ExecutionPlan for SymmetricHashJoinExec {self.input_distribution_requirements().into_per_child()}- fn input_distribution_requirements(&self) -> crate::InputDistributionRequirements {- crate::InputDistributionRequirements::new(match self.mode {+ fn input_distribution_requirements(&self) -> InputDistributionRequirements {+ match self.mode {StreamJoinPartitionMode::Partitioned => {let (left_expr, right_expr) = self.on.iter().map(|(l, r)| (Arc::clone(l) as _, Arc::clone(r) as _)).unzip();- vec![+ InputDistributionRequirements::co_partitioned(vec![Distribution::KeyPartitioned(left_expr),Distribution::KeyPartitioned(right_expr),- ]+ ])+ .allow_range_satisfaction_for_key_partitioning()}StreamJoinPartitionMode::SinglePartition => {- vec![Distribution::SinglePartition, Distribution::SinglePartition]+ InputDistributionRequirements::new(vec![+ Distribution::SinglePartition,+ Distribution::SinglePartition,+ ])}- })+ }}fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rsindex cdf4b08f71..61a9b9cc6d 100644--- a/datafusion/physical-plan/src/streaming.rs+++ b/datafusion/physical-plan/src/streaming.rs@@ -35,6 +35,7 @@ use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};use arrow::datatypes::{Schema, SchemaRef};use datafusion_common::{Result, internal_err, plan_err};use datafusion_execution::TaskContext;+use datafusion_physical_expr::projection::ProjectionMapping;use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};use async_trait::async_trait;@@ -100,7 +101,7 @@ impl StreamingTableExec {let cache = Self::compute_properties(Arc::clone(&projected_schema),projected_output_ordering.clone(),- &partitions,+ Partitioning::UnknownPartitioning(partitions.len()),infinite,);Ok(Self {@@ -115,6 +116,25 @@ impl StreamingTableExec {})}+ /// Declares the output partitioning of this stream.+ ///+ /// `output_partitioning` must describe this plan's current output and have+ /// the same number of partitions as the stream.+ pub fn with_output_partitioning(+ mut self,+ output_partitioning: Partitioning,+ ) -> Result<Self> {+ if output_partitioning.partition_count() != self.partitions.len() {+ return plan_err!(+ "Output partitioning has {} partitions but stream has {} partitions",+ output_partitioning.partition_count(),+ self.partitions.len()+ );+ }+ Arc::make_mut(&mut self.cache).partitioning = output_partitioning;+ Ok(self)+ }+pub fn partitions(&self) -> &Vec<Arc<dyn PartitionStream>> {&self.partitions}@@ -147,14 +167,12 @@ impl StreamingTableExec {fn compute_properties(schema: SchemaRef,orderings: Vec<LexOrdering>,- partitions: &[Arc<dyn PartitionStream>],+ output_partitioning: Partitioning,infinite: bool,) -> PlanProperties {// Calculate equivalence properties:let eq_properties = EquivalenceProperties::new_with_orderings(schema, orderings);- // Get output partitioning:- let output_partitioning = Partitioning::UnknownPartitioning(partitions.len());let boundedness = if infinite {Boundedness::Unbounded {requires_infinite_memory: false,@@ -204,6 +222,16 @@ impl DisplayAs for StreamingTableExec {if let Some(fetch) = self.limit {write!(f, ", fetch={fetch}")?;}+ if !matches!(+ self.cache.output_partitioning(),+ Partitioning::UnknownPartitioning(_)+ ) {+ write!(+ f,+ ", output_partitioning={}",+ self.cache.output_partitioning()+ )?;+ }display_orderings(f, &self.projected_output_ordering)?;@@ -306,6 +334,17 @@ impl ExecutionPlan for StreamingTableExec {};lex_orderings.push(ordering);}+ let projection_mapping = ProjectionMapping::try_new(+ projection+ .expr()+ .iter()+ .map(|expr| (Arc::clone(&expr.expr), expr.alias.clone())),+ &self.schema(),+ )?;+ let output_partitioning = self+ .cache+ .output_partitioning()+ .project(&projection_mapping, self.cache.equivalence_properties());StreamingTableExec::try_new(Arc::clone(self.partition_schema()),@@ -315,6 +354,7 @@ impl ExecutionPlan for StreamingTableExec {self.is_infinite(),self.limit(),)+ .and_then(|exec| exec.with_output_partitioning(output_partitioning)).map(|e| Some(Arc::new(e) as _))}diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex aa741dded7..4c8545feca 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -19,13 +19,23 @@ use std::fs::{create_dir_all, remove_dir_all, write};use std::path::Path;use std::sync::Arc;-use arrow::datatypes::{DataType, Field, Schema};+use arrow::array::{ArrayRef, Int32Array};+use arrow::compute::SortOptions;+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use arrow::record_batch::RecordBatch;+use datafusion::catalog::streaming::StreamingTable;use datafusion::common::{ScalarValue, SplitPoint};use datafusion::datasource::file_format::csv::CsvFormat;use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};use datafusion::logical_expr::{Partitioning, RangePartitioning, col};+use datafusion::physical_expr::{+ Partitioning as PhysicalPartitioning, PhysicalSortExpr,+ RangePartitioning as PhysicalRangePartitioning, expressions::col as physical_col,+};+use datafusion::physical_plan::streaming::PartitionStream;+use datafusion::physical_plan::test::TestPartitionStream;use datafusion::prelude::SessionContext;// ==============================================================================@@ -52,11 +62,13 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {.expect("range partitioning should be valid"),);+ let range_table_dir = Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("test_files/scratch_range_partitioning/range_partitioned");+register_csv_listing_table(ctx,"range_partitioned",- Path::new(env!("CARGO_MANIFEST_DIR"))- .join("test_files/scratch_range_partitioning/range_partitioned"),+ &range_table_dir,Arc::clone(&schema),["1,1,10\n5,2,50\n",@@ -67,6 +79,31 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {Some(output_partitioning),);+ register_unbounded_range_stream_table(+ ctx,+ "unbounded_range_like",+ Arc::clone(&schema),+ [10, 20, 30],+ [+ vec![(1, 1, 10), (5, 2, 50)],+ vec![(10, 1, 100), (15, 2, 150)],+ vec![(20, 1, 200), (25, 2, 250)],+ vec![(30, 1, 300), (35, 2, 350)],+ ],+ );+ register_unbounded_range_stream_table(+ ctx,+ "unbounded_range_like_shifted",+ Arc::clone(&schema),+ [15, 20, 30],+ [+ vec![(1, 1, 10), (5, 2, 50), (10, 1, 100)],+ vec![(15, 2, 150)],+ vec![(20, 1, 200), (25, 2, 250)],+ vec![(30, 1, 300), (35, 2, 350)],+ ],+ );+let shifted_output_partitioning = Partitioning::Range(RangePartitioning::try_new(vec![col("range_key").sort(true, true)],@@ -133,3 +170,64 @@ fn register_csv_listing_table(ctx.register_table(name, Arc::new(table)).expect("test listing table registration should succeed");}++fn register_unbounded_range_stream_table(+ ctx: &SessionContext,+ name: &str,+ schema: Arc<Schema>,+ split_points: [i32; 3],+ partition_rows: [Vec<(i32, i32, i32)>; 4],+) {+ let output_partitioning = PhysicalPartitioning::Range(+ PhysicalRangePartitioning::try_new(+ [PhysicalSortExpr {+ expr: physical_col("range_key", &schema)+ .expect("range key should exist in stream schema"),+ options: SortOptions::default(),+ }]+ .into(),+ split_points+ .into_iter()+ .map(|value| SplitPoint::new(vec![ScalarValue::Int32(Some(value))]))+ .collect(),+ )+ .expect("range partitioning should be valid"),+ );+ let partitions = partition_rows+ .into_iter()+ .map(|rows| range_stream_partition(Arc::clone(&schema), &rows))+ .collect();++ ctx.register_table(+ name,+ Arc::new(+ StreamingTable::try_new(schema, partitions)+ .expect("range stream table should be valid")+ .with_infinite_table(true)+ .with_output_partitioning(output_partitioning),+ ),+ )+ .expect("test stream table registration should succeed");+}++fn range_stream_partition(+ schema: SchemaRef,+ rows: &[(i32, i32, i32)],+) -> Arc<dyn PartitionStream> {+ let range_key: Vec<i32> = rows.iter().map(|(range_key, _, _)| *range_key).collect();+ let non_range_key: Vec<i32> = rows+ .iter()+ .map(|(_, non_range_key, _)| *non_range_key)+ .collect();+ let value: Vec<i32> = rows.iter().map(|(_, _, value)| *value).collect();+ let batch = RecordBatch::try_new(+ schema,+ vec![+ Arc::new(Int32Array::from(range_key)) as ArrayRef,+ Arc::new(Int32Array::from(non_range_key)) as ArrayRef,+ Arc::new(Int32Array::from(value)) as ArrayRef,+ ],+ )+ .expect("range stream batch should be valid");+ Arc::new(TestPartitionStream::new_with_batches(vec![batch]))+}diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 3accde55f7..ac92e10a8e 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -628,6 +628,149 @@ ORDER BY l.range_key;30 60035 700+##########+# TEST 18: Sort Merge Join Avoids Repartition for Compatible Range Inputs+# Compatible Range inputs satisfy SortMergeJoinExec's co-partitioned+# KeyPartitioned requirements.+##########++statement ok+set datafusion.optimizer.prefer_hash_join = false;++query TT+EXPLAIN SELECT l.range_key, l.value, r.value+FROM range_partitioned l+JOIN range_partitioned r ON l.range_key = r.range_key;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]+02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]+03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]+06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false++query III+SELECT l.range_key, l.value, r.value+FROM range_partitioned l+JOIN range_partitioned r ON l.range_key = r.range_key+ORDER BY l.range_key;+----+1 10 10+5 50 50+10 100 100+15 150 150+20 200 200+25 250 250+30 300 300+35 350 350++##########+# TEST 19: Sort Merge Join Repartitions Incompatible Range Inputs+# Different Range split points do not satisfy SortMergeJoinExec's+# co-partitioned KeyPartitioned requirements.+##########++query TT+EXPLAIN SELECT l.range_key, l.value, r.value+FROM range_partitioned l+JOIN range_partitioned_shifted r ON l.range_key = r.range_key;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]+02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]+03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]+04)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+06)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]+07)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4+08)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false++query III+SELECT l.range_key, l.value, r.value+FROM range_partitioned l+JOIN range_partitioned_shifted r ON l.range_key = r.range_key+ORDER BY l.range_key;+----+1 10 10+5 50 50+10 100 100+15 150 150+20 200 200+25 250 250+30 300 300+35 350 350++statement ok+reset datafusion.optimizer.prefer_hash_join;++##########+# TEST 20: Symmetric Hash Join Avoids Repartition for Compatible Range Inputs+# Compatible Range streams satisfy SymmetricHashJoinExec's co-partitioned+# KeyPartitioned requirements.+##########++statement ok+set datafusion.optimizer.prefer_hash_join = true;++query TT+EXPLAIN SELECT l.range_key, l.value, r.value+FROM unbounded_range_like l+FULL JOIN unbounded_range_like r ON l.range_key = r.range_key;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]+02)--SymmetricHashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)]+03)----StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)+04)----StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)++query III rowsort+SELECT l.range_key, l.value, r.value+FROM unbounded_range_like l+FULL JOIN unbounded_range_like r ON l.range_key = r.range_key;+----+1 10 10+10 100 100+15 150 150+20 200 200+25 250 250+30 300 300+35 350 350+5 50 50++##########+# TEST 21: Symmetric Hash Join Repartitions Incompatible Range Inputs+# Different Range split points do not satisfy SymmetricHashJoinExec's+# co-partitioned KeyPartitioned requirements.+##########++query TT+EXPLAIN SELECT l.range_key, l.value, r.value+FROM unbounded_range_like l+FULL JOIN unbounded_range_like_shifted r ON l.range_key = r.range_key;+----+physical_plan+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]+02)--SymmetricHashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)]+03)----RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4+04)------StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)+05)----RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4+06)------StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4)++query III rowsort+SELECT l.range_key, l.value, r.value+FROM unbounded_range_like l+FULL JOIN unbounded_range_like_shifted r ON l.range_key = r.range_key;+----+1 10 10+10 100 100+15 150 150+20 200 200+25 250 250+30 300 300+35 350 350+5 50 50+statement okreset datafusion.optimizer.prefer_hash_join;@@ -638,7 +781,7 @@ statement okreset datafusion.optimizer.preserve_file_partitions;##########-# TEST 18: Union of Range Partitioned Inputs+# TEST 22: Union of Range Partitioned Inputs# Each input exposes Range partitioning on range_key. These changes do not add a# cross-child Range relationship for UNION ALL.##########@@ -687,7 +830,7 @@ set datafusion.optimizer.preserve_file_partitions = 0;##########-# TEST 19: Window on Range Partition Column+# TEST 23: Window on Range Partition Column# Range([range_key]) colocates equal range_key values, so# PARTITION BY range_key is satisfied without a hash repartition.##########@@ -715,7 +858,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value) FROM r##########-# TEST 20: Unbounded-Frame Window on Range Partition Column+# TEST 24: Unbounded-Frame Window on Range Partition Column# The unbounded frame makes DataFusion use WindowAggExec instead of# BoundedWindowAggExec, which likewise reuses Range partitioning without a# hash repartition.@@ -744,7 +887,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value ROWS BE##########-# TEST 21: Window on Non-Range Column Rehashes+# TEST 25: Window on Non-Range Column Rehashes# Range([range_key]) does not colocate non_range_key values, so# PARTITION BY non_range_key still requires a hash repartition.##########@@ -773,7 +916,7 @@ SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER B##########-# TEST 22: Unbounded-Frame Window on Non-Range Column Rehashes+# TEST 26: Unbounded-Frame Window on Non-Range Column Rehashes# The unbounded frame makes DataFusion use WindowAggExec; Range([range_key])# does not colocate non_range_key values, so PARTITION BY non_range_key# still requires a hash repartition.@@ -803,7 +946,7 @@ SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER B##########-# TEST 23: Window Subset Satisfaction on Range Partition Column+# TEST 27: Window Subset Satisfaction on Range Partition Column# With the subset threshold met, Range([range_key]) satisfies# PARTITION BY (range_key, non_range_key): equal composite keys share the# same range_key, so they are already colocated.@@ -835,7 +978,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER B##########-# TEST 24: Window Subset Rehashes Below Subset Threshold+# TEST 28: Window Subset Rehashes Below Subset Threshold# Range([range_key]) is only a subset of PARTITION BY# (range_key, non_range_key), so it should not satisfy the window key when# subset satisfaction is disabled.@@ -874,7 +1017,7 @@ reset datafusion.optimizer.preserve_file_partitions;##########-# TEST 25: Window Without Partition Keys Uses a Single Partition+# TEST 29: Window Without Partition Keys Uses a Single Partition# A window with no PARTITION BY requires a single partition; range# partitioning is not applicable.##########@@ -904,7 +1047,7 @@ SELECT range_key, SUM(value) OVER (ORDER BY value) FROM range_partitioned ORDER##########-# TEST 26: PartitionedTopK on Range Partition Column+# TEST 30: PartitionedTopK on Range Partition Column# Exact Range([range_key]) satisfies the TopK partition key and avoids repartitioning.##########@@ -947,7 +1090,7 @@ ORDER BY range_key;##########-# TEST 27: PartitionedTopK on Non-Range Column+# TEST 31: PartitionedTopK on Non-Range Column# Partitioning on a non-range key cannot reuse Range([range_key]) and# requires hash repartitioning.##########@@ -983,7 +1126,7 @@ ORDER BY non_range_key;##########-# TEST 28: PartitionedTopK Reuses Range Subset Partitioning+# TEST 32: PartitionedTopK Reuses Range Subset Partitioning# With subset threshold met and preserve-file disabled, Range([range_key])# satisfies partitioning by (range_key, non_range_key).##########@@ -1024,7 +1167,7 @@ ORDER BY range_key, non_range_key;##########-# TEST 29: Range Subset PartitionedTopK Rehashes Below Subset Threshold+# TEST 33: Range Subset PartitionedTopK Rehashes Below Subset Threshold# Range([range_key]) is only a subset of PARTITION BY (range_key, non_range_key),# so it should not satisfy the TopK partition key when subset satisfaction is# disabled.