Skip to content

Apache DataFusion SQL Query Engine

feat: align multi-key equi-join keys with Range partitioning order (#25589)

Shayan Gh91cd7ffb927b
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index c280a0aa5e..6c00c0345d 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -21,10 +21,11 @@ use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::physical_optimizer::test_utils::{
- RequirementsTestExec, bounded_window_exec_with_can_repartition, check_integrity,
- coalesce_partitions_exec, parquet_exec_with_sort, parquet_exec_with_stats,
- repartition_exec, schema, sort_exec, sort_exec_with_preserve_partitioning,
- sort_merge_join_exec, sort_preserving_merge_exec, union_exec,
+ RequirementsTestExec, TestStreamPartition, bounded_window_exec_with_can_repartition,
+ check_integrity, coalesce_partitions_exec, parquet_exec_with_sort,
+ parquet_exec_with_stats, repartition_exec, schema, sort_exec,
+ sort_exec_with_preserve_partitioning, sort_merge_join_exec,
+ sort_preserving_merge_exec, union_exec,
};
use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array};
@@ -38,6 +39,7 @@ use datafusion::datasource::object_store::ObjectStoreUrl;
use datafusion::datasource::physical_plan::{CsvSource, ParquetSource};
use datafusion::datasource::source::DataSourceExec;
use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_common::NullEquality;
use datafusion_common::ScalarValue;
use datafusion_common::Statistics;
use datafusion_common::config::CsvOptions;
@@ -72,18 +74,21 @@ use datafusion_physical_plan::operator_statistics::{
};
use datafusion_physical_expr::{
- Distribution, Partitioning, RangePartitioning, SplitPoint,
+ AcrossPartitions, ConstExpr, Distribution, Partitioning, RangePartitioning,
+ SplitPoint,
};
use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion_physical_plan::execution_plan::ExecutionPlan;
use datafusion_physical_plan::expressions::col;
use datafusion_physical_plan::filter::FilterExec;
+use datafusion_physical_plan::joins::SortMergeJoinExec;
use datafusion_physical_plan::joins::utils::JoinOn;
use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
+use datafusion_physical_plan::streaming::StreamingTableExec;
use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
use datafusion_physical_plan::{
ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlanProperties,
@@ -423,6 +428,34 @@ fn range_partitioning(
)?))
}
+fn range_partitioning_multi(
+ keys: &[(&str, SortOptions)],
+ split_rows: &[&[i64]],
+) -> Result<Partitioning> {
+ let sort_exprs = keys
+ .iter()
+ .map(|(name, options)| Ok(PhysicalSortExpr::new(col(name, &schema())?, *options)))
+ .collect::<Result<Vec<_>>>()?;
+
+ let ordering = LexOrdering::new(sort_exprs).expect("range keys must not be empty");
+
+ let split_points = split_rows
+ .iter()
+ .map(|row| {
+ SplitPoint::new(
+ row.iter()
+ .map(|value| ScalarValue::Int64(Some(*value)))
+ .collect(),
+ )
+ })
+ .collect();
+
+ Ok(Partitioning::Range(RangePartitioning::try_new(
+ ordering,
+ split_points,
+ )?))
+}
+
fn csv_exec() -> Arc<DataSourceExec> {
csv_exec_with_sort(vec![])
}
@@ -1023,6 +1056,577 @@ fn range_inner_hash_join_rehashes_incompatible_range_partitioning() -> Result<()
Ok(())
}
+/// Optimizes `plan` under both `top_down_join_key_reordering` settings with
+/// round-robin and file-scan repartitioning disabled. Checks the expected plan,
+/// validates it with [`SanityCheckPlan`], and verifies that a second
+/// [`EnsureRequirements`] pass leaves it unchanged.
+fn check_range_join_key_alignment(
+ plan: Arc<dyn ExecutionPlan>,
+ check: impl Fn(&Arc<dyn ExecutionPlan>),
+) -> Result<()> {
+ check_range_join_key_alignment_with_partitions(4, plan, check)
+}
+
+/// [`check_range_join_key_alignment`] with `target_partitions` target partitions.
+fn check_range_join_key_alignment_with_partitions(
+ target_partitions: usize,
+ plan: Arc<dyn ExecutionPlan>,
+ check: impl Fn(&Arc<dyn ExecutionPlan>),
+) -> Result<()> {
+ for top_down in [true, false] {
+ let mut config =
+ TestConfig::default().with_query_execution_partitions(target_partitions);
+ config.config.optimizer.top_down_join_key_reordering = top_down;
+ config.config.optimizer.enable_round_robin_repartition = false;
+ config.config.optimizer.repartition_file_scans = false;
+
+ let optimized = config.try_to_plan(Arc::clone(&plan), &[Run::Distribution])?;
+ check(&optimized);
+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config.config)?;
+
+ let second = config.try_to_plan(Arc::clone(&optimized), &[Run::Distribution])?;
+ assert_eq!(
+ displayable(optimized.as_ref()).indent(true).to_string(),
+ displayable(second.as_ref()).indent(true).to_string(),
+ "plan changed on the second pass; top_down={top_down}"
+ );
+ SanityCheckPlan::new().optimize(second, &config.config)?;
+ }
+ Ok(())
+}
+
+#[test]
+fn range_join_key_alignment_reversed_keys() -> Result<()> {
+ let partitioning = range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?;
+
+ let left = parquet_exec_with_output_partitioning(partitioning.clone());
+ let right = projection_exec_with_alias(
+ parquet_exec_with_output_partitioning(partitioning),
+ vec![
+ ("a".to_string(), "a1".to_string()),
+ ("b".to_string(), "b1".to_string()),
+ ],
+ );
+
+ // Written in the opposite order of the range keys.
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b1", &right.schema())?),
+ (col("a", &left.schema())?, col("a1", &right.schema())?),
+ ];
+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_crossed_keys() -> Result<()> {
+ let left = parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?);
+
+ let right = parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("b", SortOptions::default()), ("a", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?);
+
+ // Unaliased columns crossed: left.b = right.a, left.a = right.b. The
+ // left range key list equals the *right* key list, so matching must be
+ // done per input or the permutation is missed.
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("a", &right.schema())?),
+ (col("a", &left.schema())?, col("b", &right.schema())?),
+ ];
+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1), (b@1, a@0)]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([b@1 ASC, a@0 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_sort_merge_join() -> Result<()> {
+ let asc = SortOptions::default();
+ let desc = SortOptions {
+ descending: true,
+ nulls_first: true,
+ };
+ let partitioning = range_partitioning_multi(
+ &[("a", asc), ("b", desc)],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?;
+ let left = parquet_exec_with_output_partitioning(partitioning.clone());
+ let right = parquet_exec_with_output_partitioning(partitioning);
+ let on = vec![
+ (col("b", &left.schema())?, col("b", &right.schema())?),
+ (col("a", &left.schema())?, col("a", &right.schema())?),
+ ];
+ // `sort_options` are per key pair and must be permuted with `on`; the
+ // projection must survive the rebuild.
+ let join: Arc<dyn ExecutionPlan> = Arc::new(
+ SortMergeJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ JoinType::Inner,
+ vec![desc, asc],
+ NullEquality::NullEqualsNothing,
+ )?
+ .with_projection(Some(vec![0, 5]))?,
+ );
+
+ check_range_join_key_alignment(join, |plan| {
+ let smj = plan.downcast_ref::<SortMergeJoinExec>().unwrap();
+ assert_eq!(smj.sort_options, vec![asc, desc]);
+ assert_eq!(smj.projection, Some(vec![0, 5]));
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ SortMergeJoinExec: join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@0, a@5]
+ SortExec: expr=[a@0 ASC, b@1 DESC], preserve_partitioning=[true]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 DESC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ SortExec: expr=[a@0 ASC, b@1 DESC], preserve_partitioning=[true]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 DESC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
+/// Four unbounded partitions declared as `Range([a, b])` whose rows are
+/// ordered by `ordered_by` within each partition: partition membership and
+/// row order are independent.
+fn unbounded_range_stream(ordered_by: [&str; 2]) -> Result<Arc<dyn ExecutionPlan>> {
+ ordered_stream(
+ range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?,
+ ordered_by,
+ true,
+ )
+}
+
+/// A stream table with `partitioning` whose rows are ordered by `ordered_by`
+/// within each partition.
+fn ordered_stream(
+ partitioning: Partitioning,
+ ordered_by: [&str; 2],
+ infinite: bool,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ let schema = schema();
+ let ordering: LexOrdering = [
+ PhysicalSortExpr::new(col(ordered_by[0], &schema)?, SortOptions::default()),
+ PhysicalSortExpr::new(col(ordered_by[1], &schema)?, SortOptions::default()),
+ ]
+ .into();
+ let partitions = (0..partitioning.partition_count())
+ .map(|_| {
+ Arc::new(TestStreamPartition {
+ schema: Arc::clone(&schema),
+ }) as _
+ })
+ .collect();
+ let exec = StreamingTableExec::try_new(
+ Arc::clone(&schema),
+ partitions,
+ None,
+ vec![ordering],
+ infinite,
+ None,
+ )?
+ .with_output_partitioning(partitioning)?;
+ Ok(Arc::new(exec))
+}
+
+/// Passes its input through and additionally declares `column` constant
+/// within each partition, with values that may differ across partitions.
+#[derive(Debug)]
+struct PerPartitionConstantExec {
+ input: Arc<dyn ExecutionPlan>,
+ column: String,
+ cache: Arc<PlanProperties>,
+}
+
+impl PerPartitionConstantExec {
+ fn try_new(input: Arc<dyn ExecutionPlan>, column: &str) -> Result<Self> {
+ let mut eq_properties = input.equivalence_properties().clone();
+ eq_properties.add_constants([ConstExpr::new(
+ col(column, &input.schema())?,
+ AcrossPartitions::Heterogeneous,
+ )])?;
+ let cache = PlanProperties::new(
+ eq_properties,
+ input.output_partitioning().clone(),
+ input.pipeline_behavior(),
+ input.boundedness(),
+ );
+ Ok(Self {
+ input,
+ column: column.to_string(),
+ cache: Arc::new(cache),
+ })
+ }
+}
+
+impl DisplayAs for PerPartitionConstantExec {
+ fn fmt_as(
+ &self,
+ t: DisplayFormatType,
+ f: &mut std::fmt::Formatter,
+ ) -> std::fmt::Result {
+ match t {
+ DisplayFormatType::Default | DisplayFormatType::Verbose => {
+ write!(f, "PerPartitionConstantExec: {}", self.column)
+ }
+ DisplayFormatType::TreeRender => write!(f, ""),
+ }
+ }
+}
+
+impl ExecutionPlan for PerPartitionConstantExec {
+ fn name(&self) -> &'static str {
+ "PerPartitionConstantExec"
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ &self.cache
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![&self.input]
+ }
+
+ fn maintains_input_order(&self) -> Vec<bool> {
+ vec![true]
+ }
+
+ fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+ vec![false]
+ }
+
+ fn replace_children(
+ self: Arc<Self>,
+ mut children: Vec<Arc<dyn ExecutionPlan>>,
+ _: ReplaceChildrenOptions,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ assert_eq!(children.len(), 1);
+ Ok(Arc::new(Self::try_new(
+ children.pop().unwrap(),
+ &self.column,
+ )?))
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ self.replace_children(
+ children,
+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
+ )
+ }
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<datafusion::execution::context::TaskContext>,
+ ) -> Result<datafusion_physical_plan::SendableRecordBatchStream> {
+ unreachable!();
+ }
+}
+
+#[test]
+fn range_join_key_alignment_keeps_streamable_sort_merge_join_keys() -> Result<()> {
+ // A sort-merge join on (b, a) over unbounded Range([a, b]) inputs whose
+ // rows are ordered by `ordered_by`.
+ let smj_on_b_a = |ordered_by: [&str; 2]| -> Result<Arc<dyn ExecutionPlan>> {
+ let left = unbounded_range_stream(ordered_by)?;
+ let right = unbounded_range_stream(ordered_by)?;
+ let on = vec![
+ (col("b", &left.schema())?, col("b", &right.schema())?),
+ (col("a", &left.schema())?, col("a", &right.schema())?),
+ ];
+ Ok(Arc::new(SortMergeJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ JoinType::Inner,
+ vec![SortOptions::default(); 2],
+ NullEquality::NullEqualsNothing,
+ )?))
+ };
+
+ // Rows are sorted by the keys as written, (b, a). Realigning the keys to
+ // the range order (a, b) would require a full sort of each unbounded
+ // input, which is pipeline breaking, so the keys stay as written and the
+ // inputs are repartitioned with an order-preserving exchange instead.
+ check_range_join_key_alignment(smj_on_b_a(["b", "a"])?, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]
+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]
+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]
+ "
+ );
+ }
+ })?;
+
+ // Rows are already sorted by the range order (a, b), so realigning the
+ // keys adds no sort: the join is aligned and runs without any exchange.
+ check_range_join_key_alignment(smj_on_b_a(["a", "b"])?, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ SortMergeJoinExec: join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[a@0 ASC, b@1 ASC]
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[a@0 ASC, b@1 ASC]
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_ignores_per_partition_constants() -> Result<()> {
+ // `a` is constant within each partition of the unbounded right input, so
+ // its (b, a) row order also satisfies (a, b) locally. Realigning the keys
+ // to the left Range([a, b]) would repartition the right input, which
+ // drops that constant and forces a blocking sort, so the keys are kept.
+ let left = ordered_stream(
+ range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?,
+ ["b", "a"],
+ false,
+ )?;
+ let right: Arc<dyn ExecutionPlan> = Arc::new(PerPartitionConstantExec::try_new(
+ ordered_stream(Partitioning::UnknownPartitioning(4), ["b", "a"], true)?,
+ "a",
+ )?);
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b", &right.schema())?),
+ (col("a", &left.schema())?, col("a", &right.schema())?),
+ ];
+ let join = sort_merge_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]
+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]
+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC
+ PerPartitionConstantExec: a
+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_ordering=[b@1 ASC, a@0 ASC]
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_skips_single_partition_range() -> Result<()> {
+ // A single-partition Range layout has nothing to co-partition with, so
+ // realigning the keys would only replace the existing (b, a) order with
+ // two sorts.
+ let input = || {
+ ordered_stream(
+ range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[],
+ )?,
+ ["b", "a"],
+ false,
+ )
+ };
+ let left = input()?;
+ let right = input()?;
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b", &right.schema())?),
+ (col("a", &left.schema())?, col("a", &right.schema())?),
+ ];
+ let join = sort_merge_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment_with_partitions(1, join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]
+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [], 1), output_ordering=[b@1 ASC, a@0 ASC]
+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [], 1), output_ordering=[b@1 ASC, a@0 ASC]
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_right_input_only() -> Result<()> {
+ // The left input has no Range layout, so the keys follow the right input
+ // and only the left input is repartitioned.
+ let left =
+ parquet_exec_with_output_partitioning(Partitioning::UnknownPartitioning(4));
+ let right = projection_exec_with_alias(
+ parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?),
+ vec![
+ ("a".to_string(), "a1".to_string()),
+ ("b".to_string(), "b1".to_string()),
+ ],
+ );
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b1", &right.schema())?),
+ (col("a", &left.schema())?, col("a1", &right.schema())?),
+ ];
+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]
+ RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=UnknownPartitioning(4), file_type=parquet
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_equivalent_keys() -> Result<()> {
+ // `a = c` holds on the left, so the join key `c` matches the range key `a`
+ // only through the input's equivalence classes. The right input has no
+ // Range layout, so the alignment can only come from the left.
+ let predicate = Arc::new(BinaryExpr::new(
+ col("a", &schema())?,
+ Operator::Eq,
+ col("c", &schema())?,
+ ));
+ let left: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(
+ predicate,
+ parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?),
+ )?);
+ let right = projection_exec_with_alias(
+ parquet_exec_with_output_partitioning(Partitioning::UnknownPartitioning(4)),
+ vec![
+ ("a".to_string(), "a1".to_string()),
+ ("b".to_string(), "b1".to_string()),
+ ],
+ );
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b1", &right.schema())?),
+ (col("c", &left.schema())?, col("a1", &right.schema())?),
+ ];
+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, a1@0), (b@1, b1@1)]
+ FilterExec: a@0 = c@2
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ RepartitionExec: partitioning=Range([a1@0 ASC, b1@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=UnknownPartitioning(4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
+#[test]
+fn range_join_key_alignment_swapped_range_keys_still_repartition() -> Result<()> {
+ // Same inputs as the crossed-key test, but joined b = b and a = a. The
+ // layouts share split values yet are not co-partitioned for these pairs:
+ // a row with a = 10, b = 20 is in partition 1 under Range([a, b]) and in
+ // partition 2 under Range([b, a]). Aligning the keys to the left input
+ // must leave the right input unsatisfied so it is still repartitioned.
+ let left = parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("a", SortOptions::default()), ("b", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?);
+
+ let right = parquet_exec_with_output_partitioning(range_partitioning_multi(
+ &[("b", SortOptions::default()), ("a", SortOptions::default())],
+ &[&[10, 0], &[20, 0], &[30, 0]],
+ )?);
+
+ let join_on: JoinOn = vec![
+ (col("b", &left.schema())?, col("b", &right.schema())?),
+ (col("a", &left.schema())?, col("a", &right.schema())?),
+ ];
+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);
+
+ check_range_join_key_alignment(join, |plan| {
+ insta::allow_duplicates! {
+ assert_plan!(
+ plan,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4
+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([b@1 ASC, a@0 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet
+ "
+ );
+ }
+ })
+}
+
#[test]
fn range_right_mark_hash_join_reuses_range_partitioning() -> Result<()> {
let left = parquet_exec_with_output_partitioning(range_partitioning(
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
index 912903f318..49776d8843 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs
@@ -48,7 +48,7 @@ use datafusion_physical_expr::expressions::{Column, NoOp};
use datafusion_physical_expr::utils::map_columns_before_projection;
use datafusion_physical_expr::{
EquivalenceProperties, OrderingRequirements, PhysicalExpr, PhysicalExprRef,
- physical_exprs_equal,
+ PhysicalSortExpr, physical_exprs_equal,
};
use datafusion_physical_plan::ExecutionPlanProperties;
use datafusion_physical_plan::aggregates::{
@@ -444,10 +444,10 @@ fn shift_right_required(
/// bottom left join on(b, a, c)
/// bottom right join on(c, b, a)
///
-/// Compared to the Top-Down reordering process, this Bottom-Up approach is much simpler, but might not reach a best result.
-/// The Bottom-Up approach will be useful in future if we plan to support storage partition-wised Joins.
-/// In that case, the datasources/tables might be pre-partitioned and we can't adjust the key ordering of the datasources
-/// and then can't apply the Top-Down reordering process.
+/// This bottom-up pass aligns join keys with existing hash partitioning.
+/// Range alignment happens in [`ensure_distribution_with_stats`], independently
+/// of the configured join-key reordering strategy, using the children's
+/// partitioning after bottom-up enforcement.
pub fn reorder_join_keys_to_inputs(
plan: Arc<dyn ExecutionPlan>,
) -> Result<Arc<dyn ExecutionPlan>> {
@@ -554,6 +554,159 @@ fn reorder_current_join_keys(
}
}
+/// Finds the permutation that aligns this input's join keys with its range keys.
+fn range_join_key_positions(
+ input: &dyn ExecutionPlan,
+ join_keys: &[PhysicalExprRef],
+) -> Option<Vec<usize>> {
+ let Partitioning::Range(range) = input.output_partitioning() else {
+ return None;
+ };
+
+ if range.partition_count() <= 1 || join_keys.len() != range.ordering().len() {
+ return None;
+ }
+
+ let expected = range
+ .ordering()
+ .iter()
+ .map(|sort_expr| Arc::clone(&sort_expr.expr))
+ .collect::<Vec<_>>();
+
+ expected_expr_positions(join_keys, &expected).or_else(|| {
+ let eq_group = input.equivalence_properties().eq_group();
+ if eq_group.is_empty() {
+ return None;
+ }
+
+ let normalized_keys = join_keys
+ .iter()
+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))
+ .collect::<Vec<_>>();
+ let normalized_expected = expected
+ .iter()
+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))
+ .collect::<Vec<_>>();
+
+ expected_expr_positions(&normalized_keys, &normalized_expected)
+ })
+}
+
+/// Returns true when an unbounded input does not already satisfy the
+/// proposed ordering. Conservatively avoid adding sorting requirements
+/// to unbounded inputs during join-key alignment.
+fn forces_unbounded_sort<'a>(
+ input: &Arc<dyn ExecutionPlan>,
+ keys: impl Iterator<Item = &'a PhysicalExprRef>,
+ sort_options: &[SortOptions],
+) -> Result<bool> {
+ if !input.boundedness().is_unbounded() {
+ return Ok(false);
+ }
+ let sort_exprs = keys
+ .zip(sort_options)
+ .map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options));
+ // A repartition of a multi-partition input drops its per-partition
+ // constants, so do not rely on them to satisfy the ordering.
+ let mut eq_properties = input.equivalence_properties().clone();
+ if input.output_partitioning().partition_count() > 1 {
+ eq_properties.clear_per_partition_constants();
+ }
+ Ok(!eq_properties.ordering_satisfy(sort_exprs)?)
+}
+
+/// Returns the non-identity permutation that aligns `on` with a
+/// range-partitioned input, trying the left input first and then the right.
+fn range_aligned_join_key_positions(
+ left: &Arc<dyn ExecutionPlan>,
+ right: &Arc<dyn ExecutionPlan>,
+ on: &[(PhysicalExprRef, PhysicalExprRef)],
+) -> Option<Vec<usize>> {
+ if !matches!(left.output_partitioning(), Partitioning::Range(_))
+ && !matches!(right.output_partitioning(), Partitioning::Range(_))
+ {
+ return None;
+ }
+
+ let keys = extract_join_keys(on);
+ let positions = range_join_key_positions(left.as_ref(), &keys.left_keys)
+ .or_else(|| range_join_key_positions(right.as_ref(), &keys.right_keys))?;
+
+ if positions.iter().copied().eq(0..positions.len()) {
+ return None;
+ }
+
+ Some(positions)
+}
+
+/// Aligns a partitioned join's equi-key pairs with an existing input range
+/// ordering, so compatible Range inputs satisfy the join's co-partitioning
+/// requirement without a repartition. Covers partitioned hash joins and
+/// sort-merge joins. The permutation moves whole key pairs and, for
+/// sort-merge joins, the matching `sort_options`.
+fn reorder_join_keys_to_range_inputs(
+ plan: Arc<dyn ExecutionPlan>,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ if let Some(join) = plan.downcast_ref::<HashJoinExec>() {
+ if join.mode != PartitionMode::Partitioned {
+ return Ok(plan);
+ }
+ let Some(positions) =
+ range_aligned_join_key_positions(&join.left, &join.right, &join.on)
+ else {
+ return Ok(plan);
+ };
+ let new_on = positions
+ .into_iter()
+ .map(|index| join.on[index].clone())
+ .collect();
+ return join.builder().with_on(new_on).build_exec();
+ }
+
+ if let Some(join) = plan.downcast_ref::<SortMergeJoinExec>() {
+ let Some(positions) =
+ range_aligned_join_key_positions(&join.left, &join.right, &join.on)
+ else {
+ return Ok(plan);
+ };
+ let new_on: Vec<_> = positions
+ .iter()
+ .map(|&index| join.on[index].clone())
+ .collect();
+ let new_sort_options: Vec<_> = positions
+ .iter()
+ .map(|&index| join.sort_options[index])
+ .collect();
+ // Keep the original key order unless every unbounded input already
+ // satisfies the reordered requirements, avoiding a potentially blocking sort.
+ if forces_unbounded_sort(
+ &join.left,
+ new_on.iter().map(|(left_key, _)| left_key),
+ &new_sort_options,
+ )? || forces_unbounded_sort(
+ &join.right,
+ new_on.iter().map(|(_, right_key)| right_key),
+ &new_sort_options,
+ )? {
+ return Ok(plan);
+ }
+ // `try_new` resets `projection`; restore it so the output schema is unchanged.
+ let reordered = SortMergeJoinExec::try_new(
+ Arc::clone(&join.left),
+ Arc::clone(&join.right),
+ new_on,
+ join.filter.clone(),
+ join.join_type,
+ new_sort_options,
+ join.null_equality,
+ )?
+ .with_projection(join.projection.clone())?;
+ return Ok(Arc::new(reordered) as _);
+ }
+
+ Ok(plan)
+}
+
fn try_reorder(
join_keys: JoinKeyPairs,
expected: &[Arc<dyn PhysicalExpr>],
@@ -1348,6 +1501,12 @@ pub fn ensure_distribution(
/// can first repartition (or newly partition) at the datasources -- these
/// source partitions may be later repartitioned with additional data exchange operators.
///
+/// Before reading distribution requirements, this function aligns partitioned
+/// hash-join and sort-merge-join keys with an existing input Range ordering.
+/// Reordering whole key pairs preserves the join condition while allowing
+/// compatible Range inputs to avoid repartitioning. Distribution satisfaction
+/// remains positional, so incompatible layouts still require repartitioning.
+///
/// `stats_ctx` carries the memoization cache used to answer the child
/// statistics queries behind the repartition decisions. Share one context
/// across the whole traversal so each subtree is computed once rather than
@@ -1416,6 +1575,10 @@ pub fn ensure_distribution_with_stats(
plan = updated_window;
}
+ // Align against the children's current range partitioning before deriving
+ // requirements that could otherwise introduce unnecessary repartitioning.
+ plan = reorder_join_keys_to_range_inputs(plan)?;
+
// For joins in partitioned mode, we need exact hash matching between
// both sides, so subset partitioning logic must be disabled.
//
diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
index fe3b649c6f..2cb4f57e29 100644
--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs
+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs
@@ -71,6 +71,12 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {
&[(20, 1, 200)],
&[(30, 1, 300), (40, 4, 400)],
];
+ const COMPOSITE_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [
+ &[(1, 1, 10), (10, 0, 90)],
+ &[(10, 1, 100), (10, 1, 101), (20, 0, 190)],
+ &[(20, 1, 200), (30, 0, 290)],
+ &[(30, 1, 300), (35, 2, 350)],
+ ];
let schema = Arc::new(Schema::new(vec![
Field::new("range_key", DataType::Int32, false),
@@ -187,6 +193,35 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {
sparse_output_partitioning,
None,
);
+ let composite_partitioning = Partitioning::Range(
+ RangePartitioning::try_new(
+ vec![
+ col("range_key").sort(true, true),
+ col("non_range_key").sort(true, true),
+ ],
+ [10, 20, 30]
+ .into_iter()
+ .map(|key| {
+ SplitPoint::new(vec![
+ ScalarValue::Int32(Some(key)),
+ ScalarValue::Int32(Some(1)),
+ ])
+ })
+ .collect(),
+ )
+ .expect("composite range partitioning should be valid"),
+ );
+
+ register_parquet_listing_table(
+ ctx,
+ "range_partitioned_composite",
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("test_files/scratch_range_partitioning/range_partitioned_composite"),
+ Arc::clone(&schema),
+ range_batches(&schema, COMPOSITE_RANGE_PARTITIONS),
+ composite_partitioning,
+ None,
+ );
register_sampled_range_stream_table(ctx, schema);
}
diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.slt
index 463cd1799e..b3d2a1f442 100644
--- a/datafusion/sqllogictest/test_files/range_partitioning.slt
+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt
@@ -1962,6 +1962,47 @@ ORDER BY b.range_key;
5 50 50
20 200 200
+##########
+# TEST 49: Reorder Composite Join Keys to Match Range Partitioning
+##########
+
+statement ok
+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = false;
+
+statement ok
+set datafusion.execution.parquet.pushdown_filters = false;
+
+query TT
+EXPLAIN SELECT l.range_key, l.non_range_key, l.value, r.value
+FROM range_partitioned_composite l
+JOIN range_partitioned_composite r
+ ON l.non_range_key = r.non_range_key
+ AND l.range_key = r.range_key;
+----
+physical_plan
+01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5]
+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC, non_range_key@1 ASC], [(10, 1), (20, 1), (30, 1)], 4), file_type=parquet
+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC, non_range_key@1 ASC], [(10, 1), (20, 1), (30, 1)], 4), file_type=parquet
+
+query IIII rowsort
+SELECT l.range_key, l.non_range_key, l.value, r.value
+FROM range_partitioned_composite l
+JOIN range_partitioned_composite r
+ ON l.non_range_key = r.non_range_key
+ AND l.range_key = r.range_key;
+----
+1 1 10 10
+10 0 90 90
+10 1 100 100
+10 1 100 101
+10 1 101 100
+10 1 101 101
+20 0 190 190
+20 1 200 200
+30 0 290 290
+30 1 300 300
+35 2 350 350
+
statement ok
reset datafusion.optimizer.preserve_file_partitions;