Apache DataFusion SQL Query Engine
fix: preserve range partitioning through joins (#23584)
Edson Petry5340653f35b4
diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rsindex 2a7759a8ab..90b39f7ada 100644--- a/datafusion/physical-plan/src/joins/utils.rs+++ b/datafusion/physical-plan/src/joins/utils.rs@@ -33,7 +33,8 @@ use crate::metrics::{};use crate::projection::{ProjectionExec, ProjectionExpr};use crate::{- ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Partitioning, Statistics,+ ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Partitioning,+ RangePartitioning, Statistics,};// compatibilitypub use super::join_filter::JoinFilter;@@ -68,7 +69,7 @@ use datafusion_common::stats::Precision;use datafusion_common::utils::normalize_float_zero;use datafusion_common::{DataFusionError, JoinSide, JoinType, NullEquality, Result, SharedResult,- not_impl_err, plan_err,+ internal_datafusion_err, not_impl_err, plan_err,};use datafusion_expr::Operator;use datafusion_expr::interval_arithmetic::Interval;@@ -146,9 +147,19 @@ pub fn adjust_right_output_partitioning(Partitioning::Hash(new_exprs, *size)}Partitioning::Range(range) => {- // Range partitioning optimizer propagation is tracked in- // https://github.com/apache/datafusion/issues/22395- Partitioning::UnknownPartitioning(range.partition_count())+ let ordering = add_offset_to_physical_sort_exprs(+ range.ordering().iter().cloned(),+ left_columns_len as _,+ )?;+ let ordering = LexOrdering::new(ordering).ok_or_else(|| {+ internal_datafusion_err!(+ "Offsetting range partitioning produced an empty ordering"+ )+ })?;+ Partitioning::Range(RangePartitioning::new(+ ordering,+ range.split_points().to_vec(),+ ))}result => result.clone(),};@@ -2468,7 +2479,7 @@ mod tests {use arrow::datatypes::{DataType, Fields};use arrow::error::{ArrowError, Result as ArrowResult};use datafusion_common::stats::Precision::{Absent, Exact, Inexact};- use datafusion_common::{ScalarValue, arrow_datafusion_err, arrow_err};+ use datafusion_common::{ScalarValue, SplitPoint, arrow_datafusion_err, arrow_err};use datafusion_physical_expr::PhysicalSortExpr;use rstest::rstest;@@ -4131,6 +4142,53 @@ mod tests {assert_eq!(result.column_statistics[1].byte_size, Inexact(256));}+ #[test]+ fn test_adjust_right_output_partitioning_preserves_range() -> Result<()> {+ let split_points = vec![+ SplitPoint::new(vec![+ ScalarValue::Int32(Some(10)),+ ScalarValue::Int32(Some(100)),+ ]),+ SplitPoint::new(vec![+ ScalarValue::Int32(Some(20)),+ ScalarValue::Int32(Some(50)),+ ]),+ ];+ let range = RangePartitioning::try_new(+ LexOrdering::new([+ PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions::new(false, true),+ ),+ PhysicalSortExpr::new(+ Arc::new(Column::new("b", 2)),+ SortOptions::new(true, false),+ ),+ ])+ .unwrap(),+ split_points.clone(),+ )?;++ let adjusted = adjust_right_output_partitioning(&Partitioning::Range(range), 3)?;+ let expected = Partitioning::Range(RangePartitioning::new(+ LexOrdering::new([+ PhysicalSortExpr::new(+ Arc::new(Column::new("a", 3)),+ SortOptions::new(false, true),+ ),+ PhysicalSortExpr::new(+ Arc::new(Column::new("b", 5)),+ SortOptions::new(true, false),+ ),+ ])+ .unwrap(),+ split_points,+ ));++ assert_eq!(adjusted, expected);+ Ok(())+ }+#[test]fn test_calculate_join_output_ordering() -> Result<()> {let left_ordering = LexOrdering::new(vec![diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex 6c456fe363..3accde55f7 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -506,9 +506,8 @@ set datafusion.optimizer.preserve_file_partitions = 0;########### TEST 15: Nested Range Joins-# Compatible Range partitioning satisfies the lower join inputs. The upper join-# still repairs the intermediate join output with Hash repartitioning because-# HashJoinExec does not currently expose Range output partitioning.+# Compatible Range partitioning is preserved through the lower join, allowing+# the upper join to consume it without Hash repartitioning either input.##########query TT@@ -519,12 +518,10 @@ JOIN range_partitioned s ON r.range_key = s.range_key;----physical_plan01)HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@2, range_key@0)], projection=[range_key@0, value@1, value@3, value@5]-02)--RepartitionExec: partitioning=Hash([range_key@2], 4), input_partitions=4-03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)]-04)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-05)------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-06)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-07)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)]+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+05)--DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=falsequery IIIISELECT l.range_key, l.value, r.value, s.value@@ -599,9 +596,8 @@ ORDER BY l.range_key;########### TEST 17: Range Join Feeds Aggregate-# The join inputs avoid Hash repartitioning, but the aggregate above the join-# still repartitions because HashJoinExec does not currently expose Range-# output partitioning.+# The join preserves compatible Range partitioning on range_key, allowing the+# aggregate above it to avoid Hash repartitioning.##########query TT@@ -611,12 +607,10 @@ JOIN range_partitioned r ON l.range_key = r.range_keyGROUP BY l.range_key;----physical_plan-01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key], aggr=[sum(l.value + r.value)]-02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=4-03)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(l.value + r.value)]-04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]-05)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false-06)--------DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+01)AggregateExec: mode=SinglePartitioned, gby=[range_key@0 as range_key], aggr=[sum(l.value + r.value)]+02)--HashJoinExec: mode=Partitioned, join_type=Inner, on=[(range_key@0, range_key@0)], projection=[range_key@0, value@1, value@3]+03)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+04)----DataSourceExec: file_groups=<slt:ignore>, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=falsequery IISELECT l.range_key, SUM(l.value + r.value)