Skip to content

Apache DataFusion SQL Query Engine

Enforce co-partitioning for sort merge and symmetric hash joins (#23480)

Gene Bordegaray754e11335eda
diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs
index e609877c2b..5bfecef1fb 100644
--- a/datafusion/catalog/src/streaming.rs
+++ b/datafusion/catalog/src/streaming.rs
@@ -24,7 +24,10 @@ use async_trait::async_trait;
use datafusion_common::{DFSchema, Result, plan_err};
use datafusion_expr::{Expr, SortExpr, TableType};
use datafusion_physical_expr::equivalence::project_ordering;
-use datafusion_physical_expr::{LexOrdering, create_physical_sort_exprs};
+use datafusion_physical_expr::projection::ProjectionMapping;
+use datafusion_physical_expr::{
+ EquivalenceProperties, LexOrdering, Partitioning, create_physical_sort_exprs,
+};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec};
use log::debug;
@@ -38,6 +41,7 @@ pub struct StreamingTable {
partitions: Vec<Arc<dyn PartitionStream>>,
infinite: bool,
sort_order: Vec<SortExpr>,
+ output_partitioning: Option<Partitioning>,
}
impl StreamingTable {
@@ -62,6 +66,7 @@ impl StreamingTable {
partitions,
infinite: false,
sort_order: vec![],
+ output_partitioning: None,
})
}
@@ -76,6 +81,33 @@ impl StreamingTable {
self.sort_order = sort_order;
self
}
+
+ /// Declares the output partitioning of this streaming table.
+ ///
+ /// The partitioning expressions refer to the table schema before scan
+ /// projection. If a scan projection removes a partitioning expression, the
+ /// physical plan reports unknown partitioning.
+ pub fn with_output_partitioning(mut self, output_partitioning: Partitioning) -> Self {
+ self.output_partitioning = Some(output_partitioning);
+ self
+ }
+
+ fn output_partitioning(
+ &self,
+ projection: Option<&Vec<usize>>,
+ ) -> Result<Partitioning> {
+ let Some(output_partitioning) = &self.output_partitioning else {
+ return Ok(Partitioning::UnknownPartitioning(self.partitions.len()));
+ };
+ let Some(projection) = projection else {
+ return Ok(output_partitioning.clone());
+ };
+
+ let projection_mapping =
+ ProjectionMapping::from_indices(projection, &self.schema)?;
+ let eq_properties = EquivalenceProperties::new(Arc::clone(&self.schema));
+ Ok(output_partitioning.project(&projection_mapping, &eq_properties))
+ }
}
#[async_trait]
@@ -119,13 +151,16 @@ impl TableProvider for StreamingTable {
vec![]
};
- Ok(Arc::new(StreamingTableExec::try_new(
+ let exec = StreamingTableExec::try_new(
Arc::clone(&self.schema),
self.partitions.clone(),
projection,
LexOrdering::new(physical_sort),
self.infinite,
limit,
- )?))
+ )?
+ .with_output_partitioning(self.output_partitioning(projection)?)?;
+
+ Ok(Arc::new(exec))
}
}
diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rs
index e5718f5b3d..3c426e2b09 100644
--- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs
+++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs
@@ -30,12 +30,13 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable};
use datafusion::prelude::{CsvReadOptions, SessionContext};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::{JoinType, Result, ScalarValue};
+use datafusion_common::{JoinType, NullEquality, Result, ScalarValue};
use datafusion_physical_expr::expressions::{Literal, col};
use datafusion_physical_expr::{Partitioning, RangePartitioning, SplitPoint};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::sanity_checker::SanityCheckPlan;
+use datafusion_physical_plan::joins::{StreamJoinPartitionMode, SymmetricHashJoinExec};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::{ExecutionPlan, displayable};
@@ -444,6 +445,77 @@ fn test_partitioned_hash_join_requires_co_partitioned_children() -> Result<()> {
Ok(())
}
+#[test]
+fn test_sort_merge_join_requires_co_partitioned_children() -> Result<()> {
+ let schema = create_test_schema2();
+ let join_on = vec![(col("a", &schema)?, col("a", &schema)?)];
+ let ordering: LexOrdering = [sort_expr("a", &schema)].into();
+
+ let compatible_join = sort_merge_join_exec(
+ sort_exec_with_preserve_partitioning(
+ ordering.clone(),
+ range_partitioned_exec(&schema, "a", [10])?,
+ ),
+ sort_exec_with_preserve_partitioning(
+ ordering.clone(),
+ range_partitioned_exec(&schema, "a", [10])?,
+ ),
+ &join_on,
+ &JoinType::Inner,
+ );
+ assert_sanity_check(&compatible_join, true);
+
+ let incompatible_join = sort_merge_join_exec(
+ sort_exec_with_preserve_partitioning(
+ ordering.clone(),
+ range_partitioned_exec(&schema, "a", [10])?,
+ ),
+ sort_exec_with_preserve_partitioning(
+ ordering,
+ range_partitioned_exec(&schema, "a", [20])?,
+ ),
+ &join_on,
+ &JoinType::Inner,
+ );
+ assert_sanity_check(&incompatible_join, false);
+
+ Ok(())
+}
+
+#[test]
+fn test_symmetric_hash_join_requires_co_partitioned_children() -> Result<()> {
+ let schema = create_test_schema2();
+ let join_on = vec![(col("a", &schema)?, col("a", &schema)?)];
+
+ let compatible_join = Arc::new(SymmetricHashJoinExec::try_new(
+ range_partitioned_exec(&schema, "a", [10])?,
+ range_partitioned_exec(&schema, "a", [10])?,
+ join_on.clone(),
+ None,
+ &JoinType::Inner,
+ NullEquality::NullEqualsNothing,
+ None,
+ None,
+ StreamJoinPartitionMode::Partitioned,
+ )?) as Arc<dyn ExecutionPlan>;
+ assert_sanity_check(&compatible_join, true);
+
+ let incompatible_join = Arc::new(SymmetricHashJoinExec::try_new(
+ range_partitioned_exec(&schema, "a", [10])?,
+ range_partitioned_exec(&schema, "a", [20])?,
+ join_on,
+ None,
+ &JoinType::Inner,
+ NullEquality::NullEqualsNothing,
+ None,
+ None,
+ StreamJoinPartitionMode::Partitioned,
+ )?) as Arc<dyn ExecutionPlan>;
+ assert_sanity_check(&incompatible_join, false);
+
+ Ok(())
+}
+
#[tokio::test]
/// Tests that plan is valid when the sort requirements are satisfied.
async fn test_bounded_window_agg_sort_requirement() -> Result<()> {
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index 46c37696ec..5d3621c492 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -41,7 +41,8 @@ use crate::spill::spill_manager::SpillManager;
use crate::statistics::{ChildStats, StatisticsArgs};
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
- PlanProperties, SendableRecordBatchStream, Statistics, check_if_same_properties,
+ InputDistributionRequirements, PlanProperties, SendableRecordBatchStream, Statistics,
+ check_if_same_properties,
};
use arrow::compute::SortOptions;
@@ -413,16 +414,17 @@ impl ExecutionPlan for SortMergeJoinExec {
self.input_distribution_requirements().into_per_child()
}
- fn input_distribution_requirements(&self) -> crate::InputDistributionRequirements {
+ fn input_distribution_requirements(&self) -> InputDistributionRequirements {
let (left_expr, right_expr) = self
.on
.iter()
.map(|(l, r)| (Arc::clone(l), Arc::clone(r)))
.unzip();
- crate::InputDistributionRequirements::new(vec![
+ InputDistributionRequirements::co_partitioned(vec![
Distribution::KeyPartitioned(left_expr),
Distribution::KeyPartitioned(right_expr),
])
+ .allow_range_satisfaction_for_key_partitioning()
}
fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index f33d9b1d07..0aaf8ac608 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -53,7 +53,8 @@ use crate::projection::{
use crate::stream::EmptyRecordBatchStream;
use crate::{
DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,
- PlanProperties, RecordBatchStream, SendableRecordBatchStream,
+ InputDistributionRequirements, PlanProperties, RecordBatchStream,
+ SendableRecordBatchStream,
joins::StreamJoinPartitionMode,
metrics::{ExecutionPlanMetricsSet, MetricsSet},
};
@@ -415,23 +416,27 @@ impl ExecutionPlan for SymmetricHashJoinExec {
self.input_distribution_requirements().into_per_child()
}
- fn input_distribution_requirements(&self) -> crate::InputDistributionRequirements {
- crate::InputDistributionRequirements::new(match self.mode {
+ fn input_distribution_requirements(&self) -> InputDistributionRequirements {
+ match self.mode {
StreamJoinPartitionMode::Partitioned => {
let (left_expr, right_expr) = self
.on
.iter()
.map(|(l, r)| (Arc::clone(l) as _, Arc::clone(r) as _))
.unzip();
- vec![
+ InputDistributionRequirements::co_partitioned(vec![
Distribution::KeyPartitioned(left_expr),
Distribution::KeyPartitioned(right_expr),
- ]
+ ])
+ .allow_range_satisfaction_for_key_partitioning()
}
StreamJoinPartitionMode::SinglePartition => {
- vec![Distribution::SinglePartition, Distribution::SinglePartition]
+ InputDistributionRequirements::new(vec![
+ Distribution::SinglePartition,
+ Distribution::SinglePartition,
+ ])
}
- })
+ }
}
fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {
diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs
index cdf4b08f71..61a9b9cc6d 100644
--- a/datafusion/physical-plan/src/streaming.rs
+++ b/datafusion/physical-plan/src/streaming.rs
@@ -35,6 +35,7 @@ use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};
use arrow::datatypes::{Schema, SchemaRef};
use datafusion_common::{Result, internal_err, plan_err};
use datafusion_execution::TaskContext;
+use datafusion_physical_expr::projection::ProjectionMapping;
use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};
use async_trait::async_trait;
@@ -100,7 +101,7 @@ impl StreamingTableExec {
let cache = Self::compute_properties(
Arc::clone(&projected_schema),
projected_output_ordering.clone(),
- &partitions,
+ Partitioning::UnknownPartitioning(partitions.len()),
infinite,
);
Ok(Self {
@@ -115,6 +116,25 @@ impl StreamingTableExec {
})
}
+ /// Declares the output partitioning of this stream.
+ ///
+ /// `output_partitioning` must describe this plan's current output and have
+ /// the same number of partitions as the stream.
+ pub fn with_output_partitioning(
+ mut self,
+ output_partitioning: Partitioning,
+ ) -> Result<Self> {
+ if output_partitioning.partition_count() != self.partitions.len() {
+ return plan_err!(
+ "Output partitioning has {} partitions but stream has {} partitions",
+ output_partitioning.partition_count(),
+ self.partitions.len()
+ );
+ }
+ Arc::make_mut(&mut self.cache).partitioning = output_partitioning;
+ Ok(self)
+ }
+
pub fn partitions(&self) -> &Vec<Arc<dyn PartitionStream>> {
&self.partitions
}
@@ -147,14 +167,12 @@ impl StreamingTableExec {
fn compute_properties(
schema: SchemaRef,
orderings: Vec<LexOrdering>,
- partitions: &[Arc<dyn PartitionStream>],
+ output_partitioning: Partitioning,
infinite: bool,
) -> PlanProperties {
// Calculate equivalence properties:
let eq_properties = EquivalenceProperties::new_with_orderings(schema, orderings);
- // Get output partitioning:
- let output_partitioning = Partitioning::UnknownPartitioning(partitions.len());
let boundedness = if infinite {
Boundedness::Unbounded {
requires_infinite_memory: false,
@@ -204,6 +222,16 @@ impl DisplayAs for StreamingTableExec {
if let Some(fetch) = self.limit {
write!(f, ", fetch={fetch}")?;
}
+ if !matches!(
+ self.cache.output_partitioning(),
+ Partitioning::UnknownPartitioning(_)
+ ) {
+ write!(
+ f,
+ ", output_partitioning={}",
+ self.cache.output_partitioning()
+ )?;
+ }
display_orderings(f, &self.projected_output_ordering)?;
@@ -306,6 +334,17 @@ impl ExecutionPlan for StreamingTableExec {
};
lex_orderings.push(ordering);
}
+ let projection_mapping = ProjectionMapping::try_new(
+ projection
+ .expr()
+ .iter()
+ .map(|expr| (Arc::clone(&expr.expr), expr.alias.clone())),
+ &self.schema(),
+ )?;
+ let output_partitioning = self
+ .cache
+ .output_partitioning()
+ .project(&projection_mapping, self.cache.equivalence_properties());
StreamingTableExec::try_new(
Arc::clone(self.partition_schema()),
@@ -315,6 +354,7 @@ impl ExecutionPlan for StreamingTableExec {
self.is_infinite(),
self.limit(),
)
+ .and_then(|exec| exec.with_output_partitioning(output_partitioning))
.map(|e| Some(Arc::new(e) as _))
}
diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
index aa741dded7..4c8545feca 100644
--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs
+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
@@ -19,13 +19,23 @@ use std::fs::{create_dir_all, remove_dir_all, write};
use std::path::Path;
use std::sync::Arc;
-use arrow::datatypes::{DataType, Field, Schema};
+use arrow::array::{ArrayRef, Int32Array};
+use arrow::compute::SortOptions;
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use arrow::record_batch::RecordBatch;
+use datafusion::catalog::streaming::StreamingTable;
use datafusion::common::{ScalarValue, SplitPoint};
use datafusion::datasource::file_format::csv::CsvFormat;
use datafusion::datasource::listing::{
ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
};
use datafusion::logical_expr::{Partitioning, RangePartitioning, col};
+use datafusion::physical_expr::{
+ Partitioning as PhysicalPartitioning, PhysicalSortExpr,
+ RangePartitioning as PhysicalRangePartitioning, expressions::col as physical_col,
+};
+use datafusion::physical_plan::streaming::PartitionStream;
+use datafusion::physical_plan::test::TestPartitionStream;
use datafusion::prelude::SessionContext;
// ==============================================================================
@@ -52,11 +62,13 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {
.expect("range partitioning should be valid"),
);
+ let range_table_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("test_files/scratch_range_partitioning/range_partitioned");
+
register_csv_listing_table(
ctx,
"range_partitioned",
- Path::new(env!("CARGO_MANIFEST_DIR"))
- .join("test_files/scratch_range_partitioning/range_partitioned"),
+ &range_table_dir,
Arc::clone(&schema),
[
"1,1,10\n5,2,50\n",
@@ -67,6 +79,31 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {
Some(output_partitioning),
);
+ register_unbounded_range_stream_table(
+ ctx,
+ "unbounded_range_like",
+ Arc::clone(&schema),
+ [10, 20, 30],
+ [
+ vec![(1, 1, 10), (5, 2, 50)],
+ vec![(10, 1, 100), (15, 2, 150)],
+ vec![(20, 1, 200), (25, 2, 250)],
+ vec![(30, 1, 300), (35, 2, 350)],
+ ],
+ );
+ register_unbounded_range_stream_table(
+ ctx,
+ "unbounded_range_like_shifted",
+ Arc::clone(&schema),
+ [15, 20, 30],
+ [
+ vec![(1, 1, 10), (5, 2, 50), (10, 1, 100)],
+ vec![(15, 2, 150)],
+ vec![(20, 1, 200), (25, 2, 250)],
+ vec![(30, 1, 300), (35, 2, 350)],
+ ],
+ );
+
let shifted_output_partitioning = Partitioning::Range(
RangePartitioning::try_new(
vec![col("range_key").sort(true, true)],
@@ -133,3 +170,64 @@ fn register_csv_listing_table(
ctx.register_table(name, Arc::new(table))
.expect("test listing table registration should succeed");
}
+
+fn register_unbounded_range_stream_table(
+ ctx: &SessionContext,
+ name: &str,
+ schema: Arc<Schema>,
+ split_points: [i32; 3],
+ partition_rows: [Vec<(i32, i32, i32)>; 4],
+) {
+ let output_partitioning = PhysicalPartitioning::Range(
+ PhysicalRangePartitioning::try_new(
+ [PhysicalSortExpr {
+ expr: physical_col("range_key", &schema)
+ .expect("range key should exist in stream schema"),
+ options: SortOptions::default(),
+ }]
+ .into(),
+ split_points
+ .into_iter()
+ .map(|value| SplitPoint::new(vec![ScalarValue::Int32(Some(value))]))
+ .collect(),
+ )
+ .expect("range partitioning should be valid"),
+ );
+ let partitions = partition_rows
+ .into_iter()
+ .map(|rows| range_stream_partition(Arc::clone(&schema), &rows))
+ .collect();
+
+ ctx.register_table(
+ name,
+ Arc::new(
+ StreamingTable::try_new(schema, partitions)
+ .expect("range stream table should be valid")
+ .with_infinite_table(true)
+ .with_output_partitioning(output_partitioning),
+ ),
+ )
+ .expect("test stream table registration should succeed");
+}
+
+fn range_stream_partition(
+ schema: SchemaRef,
+ rows: &[(i32, i32, i32)],
+) -> Arc<dyn PartitionStream> {
+ let range_key: Vec<i32> = rows.iter().map(|(range_key, _, _)| *range_key).collect();
+ let non_range_key: Vec<i32> = rows
+ .iter()
+ .map(|(_, non_range_key, _)| *non_range_key)
+ .collect();
+ let value: Vec<i32> = rows.iter().map(|(_, _, value)| *value).collect();
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![
+ Arc::new(Int32Array::from(range_key)) as ArrayRef,
+ Arc::new(Int32Array::from(non_range_key)) as ArrayRef,
+ Arc::new(Int32Array::from(value)) as ArrayRef,
+ ],
+ )
+ .expect("range stream batch should be valid");
+ Arc::new(TestPartitionStream::new_with_batches(vec![batch]))
+}
diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt
index 3accde55f7..ac92e10a8e 100644
--- a/datafusion/sqllogictest/test_files/range_partitioning.slt
+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt
@@ -628,6 +628,149 @@ ORDER BY l.range_key;
30 600
35 700
+##########
+# TEST 18: Sort Merge Join Avoids Repartition for Compatible Range Inputs
+# Compatible Range inputs satisfy SortMergeJoinExec's co-partitioned
+# KeyPartitioned requirements.
+##########
+
+statement ok
+set datafusion.optimizer.prefer_hash_join = false;
+
+query TT
+EXPLAIN SELECT l.range_key, l.value, r.value
+FROM range_partitioned l
+JOIN range_partitioned r ON l.range_key = r.range_key;
+----
+physical_plan
+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]
+02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]
+03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]
+04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false
+05)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]
+06)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false
+
+query III
+SELECT l.range_key, l.value, r.value
+FROM range_partitioned l
+JOIN range_partitioned r ON l.range_key = r.range_key
+ORDER BY l.range_key;
+----
+1 10 10
+5 50 50
+10 100 100
+15 150 150
+20 200 200
+25 250 250
+30 300 300
+35 350 350
+
+##########
+# TEST 19: Sort Merge Join Repartitions Incompatible Range Inputs
+# Different Range split points do not satisfy SortMergeJoinExec's
+# co-partitioned KeyPartitioned requirements.
+##########
+
+query TT
+EXPLAIN SELECT l.range_key, l.value, r.value
+FROM range_partitioned l
+JOIN range_partitioned_shifted r ON l.range_key = r.range_key;
+----
+physical_plan
+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]
+02)--SortMergeJoinExec: join_type=Inner, on=[(range_key@0, range_key@0)]
+03)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]
+04)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4
+05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false
+06)----SortExec: expr=[range_key@0 ASC], preserve_partitioning=[true]
+07)------RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4
+08)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4), file_type=csv, has_header=false
+
+query III
+SELECT l.range_key, l.value, r.value
+FROM range_partitioned l
+JOIN range_partitioned_shifted r ON l.range_key = r.range_key
+ORDER BY l.range_key;
+----
+1 10 10
+5 50 50
+10 100 100
+15 150 150
+20 200 200
+25 250 250
+30 300 300
+35 350 350
+
+statement ok
+reset datafusion.optimizer.prefer_hash_join;
+
+##########
+# TEST 20: Symmetric Hash Join Avoids Repartition for Compatible Range Inputs
+# Compatible Range streams satisfy SymmetricHashJoinExec's co-partitioned
+# KeyPartitioned requirements.
+##########
+
+statement ok
+set datafusion.optimizer.prefer_hash_join = true;
+
+query TT
+EXPLAIN SELECT l.range_key, l.value, r.value
+FROM unbounded_range_like l
+FULL JOIN unbounded_range_like r ON l.range_key = r.range_key;
+----
+physical_plan
+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]
+02)--SymmetricHashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)]
+03)----StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)
+04)----StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)
+
+query III rowsort
+SELECT l.range_key, l.value, r.value
+FROM unbounded_range_like l
+FULL JOIN unbounded_range_like r ON l.range_key = r.range_key;
+----
+1 10 10
+10 100 100
+15 150 150
+20 200 200
+25 250 250
+30 300 300
+35 350 350
+5 50 50
+
+##########
+# TEST 21: Symmetric Hash Join Repartitions Incompatible Range Inputs
+# Different Range split points do not satisfy SymmetricHashJoinExec's
+# co-partitioned KeyPartitioned requirements.
+##########
+
+query TT
+EXPLAIN SELECT l.range_key, l.value, r.value
+FROM unbounded_range_like l
+FULL JOIN unbounded_range_like_shifted r ON l.range_key = r.range_key;
+----
+physical_plan
+01)ProjectionExec: expr=[range_key@0 as range_key, value@1 as value, value@3 as value]
+02)--SymmetricHashJoinExec: mode=Partitioned, join_type=Full, on=[(range_key@0, range_key@0)]
+03)----RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4
+04)------StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)
+05)----RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4
+06)------StreamingTableExec: partition_sizes=4, projection=[range_key, value], infinite_source=true, output_partitioning=Range([range_key@0 ASC], [(15), (20), (30)], 4)
+
+query III rowsort
+SELECT l.range_key, l.value, r.value
+FROM unbounded_range_like l
+FULL JOIN unbounded_range_like_shifted r ON l.range_key = r.range_key;
+----
+1 10 10
+10 100 100
+15 150 150
+20 200 200
+25 250 250
+30 300 300
+35 350 350
+5 50 50
+
statement ok
reset datafusion.optimizer.prefer_hash_join;
@@ -638,7 +781,7 @@ statement ok
reset datafusion.optimizer.preserve_file_partitions;
##########
-# TEST 18: Union of Range Partitioned Inputs
+# TEST 22: Union of Range Partitioned Inputs
# Each input exposes Range partitioning on range_key. These changes do not add a
# cross-child Range relationship for UNION ALL.
##########
@@ -687,7 +830,7 @@ set datafusion.optimizer.preserve_file_partitions = 0;
##########
-# TEST 19: Window on Range Partition Column
+# TEST 23: Window on Range Partition Column
# Range([range_key]) colocates equal range_key values, so
# PARTITION BY range_key is satisfied without a hash repartition.
##########
@@ -715,7 +858,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value) FROM r
##########
-# TEST 20: Unbounded-Frame Window on Range Partition Column
+# TEST 24: Unbounded-Frame Window on Range Partition Column
# The unbounded frame makes DataFusion use WindowAggExec instead of
# BoundedWindowAggExec, which likewise reuses Range partitioning without a
# hash repartition.
@@ -744,7 +887,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key ORDER BY value ROWS BE
##########
-# TEST 21: Window on Non-Range Column Rehashes
+# TEST 25: Window on Non-Range Column Rehashes
# Range([range_key]) does not colocate non_range_key values, so
# PARTITION BY non_range_key still requires a hash repartition.
##########
@@ -773,7 +916,7 @@ SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER B
##########
-# TEST 22: Unbounded-Frame Window on Non-Range Column Rehashes
+# TEST 26: Unbounded-Frame Window on Non-Range Column Rehashes
# The unbounded frame makes DataFusion use WindowAggExec; Range([range_key])
# does not colocate non_range_key values, so PARTITION BY non_range_key
# still requires a hash repartition.
@@ -803,7 +946,7 @@ SELECT non_range_key, value, SUM(value) OVER (PARTITION BY non_range_key ORDER B
##########
-# TEST 23: Window Subset Satisfaction on Range Partition Column
+# TEST 27: Window Subset Satisfaction on Range Partition Column
# With the subset threshold met, Range([range_key]) satisfies
# PARTITION BY (range_key, non_range_key): equal composite keys share the
# same range_key, so they are already colocated.
@@ -835,7 +978,7 @@ SELECT range_key, SUM(value) OVER (PARTITION BY range_key, non_range_key ORDER B
##########
-# TEST 24: Window Subset Rehashes Below Subset Threshold
+# TEST 28: Window Subset Rehashes Below Subset Threshold
# Range([range_key]) is only a subset of PARTITION BY
# (range_key, non_range_key), so it should not satisfy the window key when
# subset satisfaction is disabled.
@@ -874,7 +1017,7 @@ reset datafusion.optimizer.preserve_file_partitions;
##########
-# TEST 25: Window Without Partition Keys Uses a Single Partition
+# TEST 29: Window Without Partition Keys Uses a Single Partition
# A window with no PARTITION BY requires a single partition; range
# partitioning is not applicable.
##########
@@ -904,7 +1047,7 @@ SELECT range_key, SUM(value) OVER (ORDER BY value) FROM range_partitioned ORDER
##########
-# TEST 26: PartitionedTopK on Range Partition Column
+# TEST 30: PartitionedTopK on Range Partition Column
# Exact Range([range_key]) satisfies the TopK partition key and avoids repartitioning.
##########
@@ -947,7 +1090,7 @@ ORDER BY range_key;
##########
-# TEST 27: PartitionedTopK on Non-Range Column
+# TEST 31: PartitionedTopK on Non-Range Column
# Partitioning on a non-range key cannot reuse Range([range_key]) and
# requires hash repartitioning.
##########
@@ -983,7 +1126,7 @@ ORDER BY non_range_key;
##########
-# TEST 28: PartitionedTopK Reuses Range Subset Partitioning
+# TEST 32: PartitionedTopK Reuses Range Subset Partitioning
# With subset threshold met and preserve-file disabled, Range([range_key])
# satisfies partitioning by (range_key, non_range_key).
##########
@@ -1024,7 +1167,7 @@ ORDER BY range_key, non_range_key;
##########
-# TEST 29: Range Subset PartitionedTopK Rehashes Below Subset Threshold
+# TEST 33: Range Subset PartitionedTopK Rehashes Below Subset Threshold
# Range([range_key]) is only a subset of PARTITION BY (range_key, non_range_key),
# so it should not satisfy the TopK partition key when subset satisfaction is
# disabled.