Apache DataFusion SQL Query Engine
feat: align multi-key equi-join keys with Range partitioning order (#25589)
Shayan Gh91cd7ffb927b
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex c280a0aa5e..6c00c0345d 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -21,10 +21,11 @@ use std::sync::Arc;use std::sync::atomic::{AtomicUsize, Ordering};use crate::physical_optimizer::test_utils::{- RequirementsTestExec, bounded_window_exec_with_can_repartition, check_integrity,- coalesce_partitions_exec, parquet_exec_with_sort, parquet_exec_with_stats,- repartition_exec, schema, sort_exec, sort_exec_with_preserve_partitioning,- sort_merge_join_exec, sort_preserving_merge_exec, union_exec,+ RequirementsTestExec, TestStreamPartition, bounded_window_exec_with_can_repartition,+ check_integrity, coalesce_partitions_exec, parquet_exec_with_sort,+ parquet_exec_with_stats, repartition_exec, schema, sort_exec,+ sort_exec_with_preserve_partitioning, sort_merge_join_exec,+ sort_preserving_merge_exec, union_exec,};use arrow::array::{Int64Array, RecordBatch, UInt8Array, UInt64Array};@@ -38,6 +39,7 @@ use datafusion::datasource::object_store::ObjectStoreUrl;use datafusion::datasource::physical_plan::{CsvSource, ParquetSource};use datafusion::datasource::source::DataSourceExec;use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_common::NullEquality;use datafusion_common::ScalarValue;use datafusion_common::Statistics;use datafusion_common::config::CsvOptions;@@ -72,18 +74,21 @@ use datafusion_physical_plan::operator_statistics::{};use datafusion_physical_expr::{- Distribution, Partitioning, RangePartitioning, SplitPoint,+ AcrossPartitions, ConstExpr, Distribution, Partitioning, RangePartitioning,+ SplitPoint,};use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::execution_plan::ExecutionPlan;use datafusion_physical_plan::expressions::col;use datafusion_physical_plan::filter::FilterExec;+use datafusion_physical_plan::joins::SortMergeJoinExec;use datafusion_physical_plan::joins::utils::JoinOn;use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;+use datafusion_physical_plan::streaming::StreamingTableExec;use datafusion_physical_plan::union::{InterleaveExec, UnionExec};use datafusion_physical_plan::{ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlanProperties,@@ -423,6 +428,34 @@ fn range_partitioning()?))}+fn range_partitioning_multi(+ keys: &[(&str, SortOptions)],+ split_rows: &[&[i64]],+) -> Result<Partitioning> {+ let sort_exprs = keys+ .iter()+ .map(|(name, options)| Ok(PhysicalSortExpr::new(col(name, &schema())?, *options)))+ .collect::<Result<Vec<_>>>()?;++ let ordering = LexOrdering::new(sort_exprs).expect("range keys must not be empty");++ let split_points = split_rows+ .iter()+ .map(|row| {+ SplitPoint::new(+ row.iter()+ .map(|value| ScalarValue::Int64(Some(*value)))+ .collect(),+ )+ })+ .collect();++ Ok(Partitioning::Range(RangePartitioning::try_new(+ ordering,+ split_points,+ )?))+}+fn csv_exec() -> Arc<DataSourceExec> {csv_exec_with_sort(vec![])}@@ -1023,6 +1056,577 @@ fn range_inner_hash_join_rehashes_incompatible_range_partitioning() -> Result<()Ok(())}+/// Optimizes `plan` under both `top_down_join_key_reordering` settings with+/// round-robin and file-scan repartitioning disabled. Checks the expected plan,+/// validates it with [`SanityCheckPlan`], and verifies that a second+/// [`EnsureRequirements`] pass leaves it unchanged.+fn check_range_join_key_alignment(+ plan: Arc<dyn ExecutionPlan>,+ check: impl Fn(&Arc<dyn ExecutionPlan>),+) -> Result<()> {+ check_range_join_key_alignment_with_partitions(4, plan, check)+}++/// [`check_range_join_key_alignment`] with `target_partitions` target partitions.+fn check_range_join_key_alignment_with_partitions(+ target_partitions: usize,+ plan: Arc<dyn ExecutionPlan>,+ check: impl Fn(&Arc<dyn ExecutionPlan>),+) -> Result<()> {+ for top_down in [true, false] {+ let mut config =+ TestConfig::default().with_query_execution_partitions(target_partitions);+ config.config.optimizer.top_down_join_key_reordering = top_down;+ config.config.optimizer.enable_round_robin_repartition = false;+ config.config.optimizer.repartition_file_scans = false;++ let optimized = config.try_to_plan(Arc::clone(&plan), &[Run::Distribution])?;+ check(&optimized);+ SanityCheckPlan::new().optimize(Arc::clone(&optimized), &config.config)?;++ let second = config.try_to_plan(Arc::clone(&optimized), &[Run::Distribution])?;+ assert_eq!(+ displayable(optimized.as_ref()).indent(true).to_string(),+ displayable(second.as_ref()).indent(true).to_string(),+ "plan changed on the second pass; top_down={top_down}"+ );+ SanityCheckPlan::new().optimize(second, &config.config)?;+ }+ Ok(())+}++#[test]+fn range_join_key_alignment_reversed_keys() -> Result<()> {+ let partitioning = range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?;++ let left = parquet_exec_with_output_partitioning(partitioning.clone());+ let right = projection_exec_with_alias(+ parquet_exec_with_output_partitioning(partitioning),+ vec![+ ("a".to_string(), "a1".to_string()),+ ("b".to_string(), "b1".to_string()),+ ],+ );++ // Written in the opposite order of the range keys.+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b1", &right.schema())?),+ (col("a", &left.schema())?, col("a1", &right.schema())?),+ ];+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_crossed_keys() -> Result<()> {+ let left = parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?);++ let right = parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("b", SortOptions::default()), ("a", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?);++ // Unaliased columns crossed: left.b = right.a, left.a = right.b. The+ // left range key list equals the *right* key list, so matching must be+ // done per input or the permutation is missed.+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("a", &right.schema())?),+ (col("a", &left.schema())?, col("b", &right.schema())?),+ ];+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1), (b@1, a@0)]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([b@1 ASC, a@0 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_sort_merge_join() -> Result<()> {+ let asc = SortOptions::default();+ let desc = SortOptions {+ descending: true,+ nulls_first: true,+ };+ let partitioning = range_partitioning_multi(+ &[("a", asc), ("b", desc)],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?;+ let left = parquet_exec_with_output_partitioning(partitioning.clone());+ let right = parquet_exec_with_output_partitioning(partitioning);+ let on = vec![+ (col("b", &left.schema())?, col("b", &right.schema())?),+ (col("a", &left.schema())?, col("a", &right.schema())?),+ ];+ // `sort_options` are per key pair and must be permuted with `on`; the+ // projection must survive the rebuild.+ let join: Arc<dyn ExecutionPlan> = Arc::new(+ SortMergeJoinExec::try_new(+ left,+ right,+ on,+ None,+ JoinType::Inner,+ vec![desc, asc],+ NullEquality::NullEqualsNothing,+ )?+ .with_projection(Some(vec![0, 5]))?,+ );++ check_range_join_key_alignment(join, |plan| {+ let smj = plan.downcast_ref::<SortMergeJoinExec>().unwrap();+ assert_eq!(smj.sort_options, vec![asc, desc]);+ assert_eq!(smj.projection, Some(vec![0, 5]));+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ SortMergeJoinExec: join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@0, a@5]+ SortExec: expr=[a@0 ASC, b@1 DESC], preserve_partitioning=[true]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 DESC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ SortExec: expr=[a@0 ASC, b@1 DESC], preserve_partitioning=[true]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 DESC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ "+ );+ }+ })+}++/// Four unbounded partitions declared as `Range([a, b])` whose rows are+/// ordered by `ordered_by` within each partition: partition membership and+/// row order are independent.+fn unbounded_range_stream(ordered_by: [&str; 2]) -> Result<Arc<dyn ExecutionPlan>> {+ ordered_stream(+ range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?,+ ordered_by,+ true,+ )+}++/// A stream table with `partitioning` whose rows are ordered by `ordered_by`+/// within each partition.+fn ordered_stream(+ partitioning: Partitioning,+ ordered_by: [&str; 2],+ infinite: bool,+) -> Result<Arc<dyn ExecutionPlan>> {+ let schema = schema();+ let ordering: LexOrdering = [+ PhysicalSortExpr::new(col(ordered_by[0], &schema)?, SortOptions::default()),+ PhysicalSortExpr::new(col(ordered_by[1], &schema)?, SortOptions::default()),+ ]+ .into();+ let partitions = (0..partitioning.partition_count())+ .map(|_| {+ Arc::new(TestStreamPartition {+ schema: Arc::clone(&schema),+ }) as _+ })+ .collect();+ let exec = StreamingTableExec::try_new(+ Arc::clone(&schema),+ partitions,+ None,+ vec![ordering],+ infinite,+ None,+ )?+ .with_output_partitioning(partitioning)?;+ Ok(Arc::new(exec))+}++/// Passes its input through and additionally declares `column` constant+/// within each partition, with values that may differ across partitions.+#[derive(Debug)]+struct PerPartitionConstantExec {+ input: Arc<dyn ExecutionPlan>,+ column: String,+ cache: Arc<PlanProperties>,+}++impl PerPartitionConstantExec {+ fn try_new(input: Arc<dyn ExecutionPlan>, column: &str) -> Result<Self> {+ let mut eq_properties = input.equivalence_properties().clone();+ eq_properties.add_constants([ConstExpr::new(+ col(column, &input.schema())?,+ AcrossPartitions::Heterogeneous,+ )])?;+ let cache = PlanProperties::new(+ eq_properties,+ input.output_partitioning().clone(),+ input.pipeline_behavior(),+ input.boundedness(),+ );+ Ok(Self {+ input,+ column: column.to_string(),+ cache: Arc::new(cache),+ })+ }+}++impl DisplayAs for PerPartitionConstantExec {+ fn fmt_as(+ &self,+ t: DisplayFormatType,+ f: &mut std::fmt::Formatter,+ ) -> std::fmt::Result {+ match t {+ DisplayFormatType::Default | DisplayFormatType::Verbose => {+ write!(f, "PerPartitionConstantExec: {}", self.column)+ }+ DisplayFormatType::TreeRender => write!(f, ""),+ }+ }+}++impl ExecutionPlan for PerPartitionConstantExec {+ fn name(&self) -> &'static str {+ "PerPartitionConstantExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ &self.cache+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![&self.input]+ }++ fn maintains_input_order(&self) -> Vec<bool> {+ vec![true]+ }++ fn benefits_from_input_partitioning(&self) -> Vec<bool> {+ vec![false]+ }++ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ assert_eq!(children.len(), 1);+ Ok(Arc::new(Self::try_new(+ children.pop().unwrap(),+ &self.column,+ )?))+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<datafusion::execution::context::TaskContext>,+ ) -> Result<datafusion_physical_plan::SendableRecordBatchStream> {+ unreachable!();+ }+}++#[test]+fn range_join_key_alignment_keeps_streamable_sort_merge_join_keys() -> Result<()> {+ // A sort-merge join on (b, a) over unbounded Range([a, b]) inputs whose+ // rows are ordered by `ordered_by`.+ let smj_on_b_a = |ordered_by: [&str; 2]| -> Result<Arc<dyn ExecutionPlan>> {+ let left = unbounded_range_stream(ordered_by)?;+ let right = unbounded_range_stream(ordered_by)?;+ let on = vec![+ (col("b", &left.schema())?, col("b", &right.schema())?),+ (col("a", &left.schema())?, col("a", &right.schema())?),+ ];+ Ok(Arc::new(SortMergeJoinExec::try_new(+ left,+ right,+ on,+ None,+ JoinType::Inner,+ vec![SortOptions::default(); 2],+ NullEquality::NullEqualsNothing,+ )?))+ };++ // Rows are sorted by the keys as written, (b, a). Realigning the keys to+ // the range order (a, b) would require a full sort of each unbounded+ // input, which is pipeline breaking, so the keys stay as written and the+ // inputs are repartitioned with an order-preserving exchange instead.+ check_range_join_key_alignment(smj_on_b_a(["b", "a"])?, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]+ "+ );+ }+ })?;++ // Rows are already sorted by the range order (a, b), so realigning the+ // keys adds no sort: the join is aligned and runs without any exchange.+ check_range_join_key_alignment(smj_on_b_a(["a", "b"])?, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ SortMergeJoinExec: join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[a@0 ASC, b@1 ASC]+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[a@0 ASC, b@1 ASC]+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_ignores_per_partition_constants() -> Result<()> {+ // `a` is constant within each partition of the unbounded right input, so+ // its (b, a) row order also satisfies (a, b) locally. Realigning the keys+ // to the left Range([a, b]) would repartition the right input, which+ // drops that constant and forces a blocking sort, so the keys are kept.+ let left = ordered_stream(+ range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?,+ ["b", "a"],+ false,+ )?;+ let right: Arc<dyn ExecutionPlan> = Arc::new(PerPartitionConstantExec::try_new(+ ordered_stream(Partitioning::UnknownPartitioning(4), ["b", "a"], true)?,+ "a",+ )?);+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b", &right.schema())?),+ (col("a", &left.schema())?, col("a", &right.schema())?),+ ];+ let join = sort_merge_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), output_ordering=[b@1 ASC, a@0 ASC]+ RepartitionExec: partitioning=Hash([b@1, a@0], 4), input_partitions=4, preserve_order=true, sort_exprs=b@1 ASC, a@0 ASC+ PerPartitionConstantExec: a+ StreamingTableExec: partition_sizes=4, projection=[a, b, c, d, e], infinite_source=true, output_ordering=[b@1 ASC, a@0 ASC]+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_skips_single_partition_range() -> Result<()> {+ // A single-partition Range layout has nothing to co-partition with, so+ // realigning the keys would only replace the existing (b, a) order with+ // two sorts.+ let input = || {+ ordered_stream(+ range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[],+ )?,+ ["b", "a"],+ false,+ )+ };+ let left = input()?;+ let right = input()?;+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b", &right.schema())?),+ (col("a", &left.schema())?, col("a", &right.schema())?),+ ];+ let join = sort_merge_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment_with_partitions(1, join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ SortMergeJoinExec: join_type=Inner, on=[(b@1, b@1), (a@0, a@0)]+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [], 1), output_ordering=[b@1 ASC, a@0 ASC]+ StreamingTableExec: partition_sizes=1, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [], 1), output_ordering=[b@1 ASC, a@0 ASC]+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_right_input_only() -> Result<()> {+ // The left input has no Range layout, so the keys follow the right input+ // and only the left input is repartitioned.+ let left =+ parquet_exec_with_output_partitioning(Partitioning::UnknownPartitioning(4));+ let right = projection_exec_with_alias(+ parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?),+ vec![+ ("a".to_string(), "a1".to_string()),+ ("b".to_string(), "b1".to_string()),+ ],+ );+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b1", &right.schema())?),+ (col("a", &left.schema())?, col("a1", &right.schema())?),+ ];+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]+ RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=UnknownPartitioning(4), file_type=parquet+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_equivalent_keys() -> Result<()> {+ // `a = c` holds on the left, so the join key `c` matches the range key `a`+ // only through the input's equivalence classes. The right input has no+ // Range layout, so the alignment can only come from the left.+ let predicate = Arc::new(BinaryExpr::new(+ col("a", &schema())?,+ Operator::Eq,+ col("c", &schema())?,+ ));+ let left: Arc<dyn ExecutionPlan> = Arc::new(FilterExec::try_new(+ predicate,+ parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?),+ )?);+ let right = projection_exec_with_alias(+ parquet_exec_with_output_partitioning(Partitioning::UnknownPartitioning(4)),+ vec![+ ("a".to_string(), "a1".to_string()),+ ("b".to_string(), "b1".to_string()),+ ],+ );+ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b1", &right.schema())?),+ (col("c", &left.schema())?, col("a1", &right.schema())?),+ ];+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, a1@0), (b@1, b1@1)]+ FilterExec: a@0 = c@2+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ RepartitionExec: partitioning=Range([a1@0 ASC, b1@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4+ ProjectionExec: expr=[a@0 as a1, b@1 as b1]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=UnknownPartitioning(4), file_type=parquet+ "+ );+ }+ })+}++#[test]+fn range_join_key_alignment_swapped_range_keys_still_repartition() -> Result<()> {+ // Same inputs as the crossed-key test, but joined b = b and a = a. The+ // layouts share split values yet are not co-partitioned for these pairs:+ // a row with a = 10, b = 20 is in partition 1 under Range([a, b]) and in+ // partition 2 under Range([b, a]). Aligning the keys to the left input+ // must leave the right input unsatisfied so it is still repartitioned.+ let left = parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("a", SortOptions::default()), ("b", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?);++ let right = parquet_exec_with_output_partitioning(range_partitioning_multi(+ &[("b", SortOptions::default()), ("a", SortOptions::default())],+ &[&[10, 0], &[20, 0], &[30, 0]],+ )?);++ let join_on: JoinOn = vec![+ (col("b", &left.schema())?, col("b", &right.schema())?),+ (col("a", &left.schema())?, col("a", &right.schema())?),+ ];+ let join = hash_join_exec(left, right, &join_on, &JoinType::Inner);++ check_range_join_key_alignment(join, |plan| {+ insta::allow_duplicates! {+ assert_plan!(+ plan,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)]+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ RepartitionExec: partitioning=Range([a@0 ASC, b@1 ASC], [(10, 0), (20, 0), (30, 0)], 4), input_partitions=4+ DataSourceExec: file_groups={4 groups: [[p0], [p1], [p2], [p3]]}, projection=[a, b, c, d, e], output_partitioning=Range([b@1 ASC, a@0 ASC], [(10, 0), (20, 0), (30, 0)], 4), file_type=parquet+ "+ );+ }+ })+}+#[test]fn range_right_mark_hash_join_reuses_range_partitioning() -> Result<()> {let left = parquet_exec_with_output_partitioning(range_partitioning(diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex 912903f318..49776d8843 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -48,7 +48,7 @@ use datafusion_physical_expr::expressions::{Column, NoOp};use datafusion_physical_expr::utils::map_columns_before_projection;use datafusion_physical_expr::{EquivalenceProperties, OrderingRequirements, PhysicalExpr, PhysicalExprRef,- physical_exprs_equal,+ PhysicalSortExpr, physical_exprs_equal,};use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_physical_plan::aggregates::{@@ -444,10 +444,10 @@ fn shift_right_required(/// bottom left join on(b, a, c)/// bottom right join on(c, b, a)///-/// Compared to the Top-Down reordering process, this Bottom-Up approach is much simpler, but might not reach a best result.-/// The Bottom-Up approach will be useful in future if we plan to support storage partition-wised Joins.-/// In that case, the datasources/tables might be pre-partitioned and we can't adjust the key ordering of the datasources-/// and then can't apply the Top-Down reordering process.+/// This bottom-up pass aligns join keys with existing hash partitioning.+/// Range alignment happens in [`ensure_distribution_with_stats`], independently+/// of the configured join-key reordering strategy, using the children's+/// partitioning after bottom-up enforcement.pub fn reorder_join_keys_to_inputs(plan: Arc<dyn ExecutionPlan>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -554,6 +554,159 @@ fn reorder_current_join_keys(}}+/// Finds the permutation that aligns this input's join keys with its range keys.+fn range_join_key_positions(+ input: &dyn ExecutionPlan,+ join_keys: &[PhysicalExprRef],+) -> Option<Vec<usize>> {+ let Partitioning::Range(range) = input.output_partitioning() else {+ return None;+ };++ if range.partition_count() <= 1 || join_keys.len() != range.ordering().len() {+ return None;+ }++ let expected = range+ .ordering()+ .iter()+ .map(|sort_expr| Arc::clone(&sort_expr.expr))+ .collect::<Vec<_>>();++ expected_expr_positions(join_keys, &expected).or_else(|| {+ let eq_group = input.equivalence_properties().eq_group();+ if eq_group.is_empty() {+ return None;+ }++ let normalized_keys = join_keys+ .iter()+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))+ .collect::<Vec<_>>();+ let normalized_expected = expected+ .iter()+ .map(|expr| eq_group.normalize_expr(Arc::clone(expr)))+ .collect::<Vec<_>>();++ expected_expr_positions(&normalized_keys, &normalized_expected)+ })+}++/// Returns true when an unbounded input does not already satisfy the+/// proposed ordering. Conservatively avoid adding sorting requirements+/// to unbounded inputs during join-key alignment.+fn forces_unbounded_sort<'a>(+ input: &Arc<dyn ExecutionPlan>,+ keys: impl Iterator<Item = &'a PhysicalExprRef>,+ sort_options: &[SortOptions],+) -> Result<bool> {+ if !input.boundedness().is_unbounded() {+ return Ok(false);+ }+ let sort_exprs = keys+ .zip(sort_options)+ .map(|(expr, options)| PhysicalSortExpr::new(Arc::clone(expr), *options));+ // A repartition of a multi-partition input drops its per-partition+ // constants, so do not rely on them to satisfy the ordering.+ let mut eq_properties = input.equivalence_properties().clone();+ if input.output_partitioning().partition_count() > 1 {+ eq_properties.clear_per_partition_constants();+ }+ Ok(!eq_properties.ordering_satisfy(sort_exprs)?)+}++/// Returns the non-identity permutation that aligns `on` with a+/// range-partitioned input, trying the left input first and then the right.+fn range_aligned_join_key_positions(+ left: &Arc<dyn ExecutionPlan>,+ right: &Arc<dyn ExecutionPlan>,+ on: &[(PhysicalExprRef, PhysicalExprRef)],+) -> Option<Vec<usize>> {+ if !matches!(left.output_partitioning(), Partitioning::Range(_))+ && !matches!(right.output_partitioning(), Partitioning::Range(_))+ {+ return None;+ }++ let keys = extract_join_keys(on);+ let positions = range_join_key_positions(left.as_ref(), &keys.left_keys)+ .or_else(|| range_join_key_positions(right.as_ref(), &keys.right_keys))?;++ if positions.iter().copied().eq(0..positions.len()) {+ return None;+ }++ Some(positions)+}++/// Aligns a partitioned join's equi-key pairs with an existing input range+/// ordering, so compatible Range inputs satisfy the join's co-partitioning+/// requirement without a repartition. Covers partitioned hash joins and+/// sort-merge joins. The permutation moves whole key pairs and, for+/// sort-merge joins, the matching `sort_options`.+fn reorder_join_keys_to_range_inputs(+ plan: Arc<dyn ExecutionPlan>,+) -> Result<Arc<dyn ExecutionPlan>> {+ if let Some(join) = plan.downcast_ref::<HashJoinExec>() {+ if join.mode != PartitionMode::Partitioned {+ return Ok(plan);+ }+ let Some(positions) =+ range_aligned_join_key_positions(&join.left, &join.right, &join.on)+ else {+ return Ok(plan);+ };+ let new_on = positions+ .into_iter()+ .map(|index| join.on[index].clone())+ .collect();+ return join.builder().with_on(new_on).build_exec();+ }++ if let Some(join) = plan.downcast_ref::<SortMergeJoinExec>() {+ let Some(positions) =+ range_aligned_join_key_positions(&join.left, &join.right, &join.on)+ else {+ return Ok(plan);+ };+ let new_on: Vec<_> = positions+ .iter()+ .map(|&index| join.on[index].clone())+ .collect();+ let new_sort_options: Vec<_> = positions+ .iter()+ .map(|&index| join.sort_options[index])+ .collect();+ // Keep the original key order unless every unbounded input already+ // satisfies the reordered requirements, avoiding a potentially blocking sort.+ if forces_unbounded_sort(+ &join.left,+ new_on.iter().map(|(left_key, _)| left_key),+ &new_sort_options,+ )? || forces_unbounded_sort(+ &join.right,+ new_on.iter().map(|(_, right_key)| right_key),+ &new_sort_options,+ )? {+ return Ok(plan);+ }+ // `try_new` resets `projection`; restore it so the output schema is unchanged.+ let reordered = SortMergeJoinExec::try_new(+ Arc::clone(&join.left),+ Arc::clone(&join.right),+ new_on,+ join.filter.clone(),+ join.join_type,+ new_sort_options,+ join.null_equality,+ )?+ .with_projection(join.projection.clone())?;+ return Ok(Arc::new(reordered) as _);+ }++ Ok(plan)+}+fn try_reorder(join_keys: JoinKeyPairs,expected: &[Arc<dyn PhysicalExpr>],@@ -1348,6 +1501,12 @@ pub fn ensure_distribution(/// can first repartition (or newly partition) at the datasources -- these/// source partitions may be later repartitioned with additional data exchange operators.///+/// Before reading distribution requirements, this function aligns partitioned+/// hash-join and sort-merge-join keys with an existing input Range ordering.+/// Reordering whole key pairs preserves the join condition while allowing+/// compatible Range inputs to avoid repartitioning. Distribution satisfaction+/// remains positional, so incompatible layouts still require repartitioning.+////// `stats_ctx` carries the memoization cache used to answer the child/// statistics queries behind the repartition decisions. Share one context/// across the whole traversal so each subtree is computed once rather than@@ -1416,6 +1575,10 @@ pub fn ensure_distribution_with_stats(plan = updated_window;}+ // Align against the children's current range partitioning before deriving+ // requirements that could otherwise introduce unnecessary repartitioning.+ plan = reorder_join_keys_to_range_inputs(plan)?;+// For joins in partitioned mode, we need exact hash matching between// both sides, so subset partitioning logic must be disabled.//diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex fe3b649c6f..2cb4f57e29 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -71,6 +71,12 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {&[(20, 1, 200)],&[(30, 1, 300), (40, 4, 400)],];+ const COMPOSITE_RANGE_PARTITIONS: [&[(i32, i32, i32)]; 4] = [+ &[(1, 1, 10), (10, 0, 90)],+ &[(10, 1, 100), (10, 1, 101), (20, 0, 190)],+ &[(20, 1, 200), (30, 0, 290)],+ &[(30, 1, 300), (35, 2, 350)],+ ];let schema = Arc::new(Schema::new(vec![Field::new("range_key", DataType::Int32, false),@@ -187,6 +193,35 @@ pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {sparse_output_partitioning,None,);+ let composite_partitioning = Partitioning::Range(+ RangePartitioning::try_new(+ vec![+ col("range_key").sort(true, true),+ col("non_range_key").sort(true, true),+ ],+ [10, 20, 30]+ .into_iter()+ .map(|key| {+ SplitPoint::new(vec![+ ScalarValue::Int32(Some(key)),+ ScalarValue::Int32(Some(1)),+ ])+ })+ .collect(),+ )+ .expect("composite range partitioning should be valid"),+ );++ register_parquet_listing_table(+ ctx,+ "range_partitioned_composite",+ Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("test_files/scratch_range_partitioning/range_partitioned_composite"),+ Arc::clone(&schema),+ range_batches(&schema, COMPOSITE_RANGE_PARTITIONS),+ composite_partitioning,+ None,+ );register_sampled_range_stream_table(ctx, schema);}diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 463cd1799e..b3d2a1f442 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -1962,6 +1962,47 @@ ORDER BY b.range_key;5 50 5020 200 200+##########+# TEST 49: Reorder Composite Join Keys to Match Range Partitioning+##########++statement ok+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = false;++statement ok+set datafusion.execution.parquet.pushdown_filters = false;++query TT+EXPLAIN SELECT l.range_key, l.non_range_key, l.value, r.value+FROM range_partitioned_composite l+JOIN range_partitioned_composite r+ ON l.non_range_key = r.non_range_key+ AND l.range_key = r.range_key;+----+physical_plan+01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0), (non_range_key@1, non_range_key@1)], projection=[range_key@0, non_range_key@1, value@2, value@5]+02)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC, non_range_key@1 ASC], [(10, 1), (20, 1), (30, 1)], 4), file_type=parquet+03)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, non_range_key, value], output_partitioning=Range([range_key@0 ASC, non_range_key@1 ASC], [(10, 1), (20, 1), (30, 1)], 4), file_type=parquet++query IIII rowsort+SELECT l.range_key, l.non_range_key, l.value, r.value+FROM range_partitioned_composite l+JOIN range_partitioned_composite r+ ON l.non_range_key = r.non_range_key+ AND l.range_key = r.range_key;+----+1 1 10 10+10 0 90 90+10 1 100 100+10 1 100 101+10 1 101 100+10 1 101 101+20 0 190 190+20 1 200 200+30 0 290 290+30 1 300 300+35 2 350 350+statement okreset datafusion.optimizer.preserve_file_partitions;