Apache DataFusion SQL Query Engine
fix: don't report equi-key equivalence for PiecewiseMergeJoin range joins (#24361)
Liang-Chi Hsiehcc7565be1ee9
diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rsindex a91903934a..5183d3aa0f 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs@@ -363,7 +363,6 @@ impl PiecewiseMergeJoinExec {&streamed,Arc::clone(&schema),join_type,- &on,)?;Ok(Self {@@ -425,7 +424,6 @@ impl PiecewiseMergeJoinExec {streamed: &Arc<dyn ExecutionPlan>,schema: SchemaRef,join_type: JoinType,- join_on: &(PhysicalExprRef, PhysicalExprRef),) -> Result<PlanProperties> {let eq_properties = join_equivalence_properties(buffered.equivalence_properties().clone(),@@ -434,7 +432,12 @@ impl PiecewiseMergeJoinExec {schema,&Self::maintains_input_order(join_type),Some(Self::probe_side(&join_type)),- std::slice::from_ref(join_on),+ // `PiecewiseMergeJoin`'s `on` is a range predicate (e.g. `l < r`),+ // not an equijoin key. Passing it here would register a false+ // `left == right` output equivalence, letting the optimizer drop a+ // required sort and return wrongly ordered results. Range joins add+ // no column equivalences, so pass none.+ &[],)?;let output_partitioning =diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.sltindex 42be6c6b5c..f71a1b6672 100644--- a/datafusion/sqllogictest/test_files/pwmj.slt+++ b/datafusion/sqllogictest/test_files/pwmj.slt@@ -129,7 +129,7 @@ logical_plan09)----------TableScan: join_t2 projection=[t2_id, t2_int]physical_plan01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST]-02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]+02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true]03)----PiecewiseMergeJoin: operator=GtEq, join_type=Inner, on=(t1_id >= t2_id)04)------SortExec: expr=[t1_id@0 ASC], preserve_partitioning=[false]05)--------FilterExec: t1_id@0 >= 22@@ -146,12 +146,12 @@ JOIN join_t2 t2WHERE t2.t2_int >= 3ORDER BY 1,2;-----11 5511 44-22 55+11 5522 44-33 55+22 5533 44+33 5544 55query TT@@ -174,7 +174,7 @@ logical_plan08)----------TableScan: join_t2 projection=[t2_id, t2_int]physical_plan01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST]-02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]+02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true]03)----PiecewiseMergeJoin: operator=Lt, join_type=Inner, on=(t1_id < t2_id)04)------SortExec: expr=[t1_id@0 DESC], preserve_partitioning=[false]05)--------DataSourceExec: partitions=1, partition_sizes=[1]@@ -238,11 +238,11 @@ WHERE t1.t1_id IN (11, 44)AND t2.t2_name <> 'y'ORDER BY 1,2;-----11 55-11 4411 11-44 55+11 44+11 5544 44+44 55query TTEXPLAIN@@ -266,7 +266,7 @@ logical_plan09)----------TableScan: join_t2 projection=[t2_id, t2_name]physical_plan01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST]-02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true]+02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true]03)----PiecewiseMergeJoin: operator=LtEq, join_type=Inner, on=(t1_id <= t2_id)04)------SortExec: expr=[t1_id@0 DESC], preserve_partitioning=[false]05)--------FilterExec: t1_id@0 = 11 OR t1_id@0 = 44@@ -387,6 +387,34 @@ ORDER BY 1,2;20 100NULL NULL+# Regression test: a range predicate is NOT an equi-key, so an INNER+# PiecewiseMergeJoin must not report `l.v == r.v` as an output equivalence.+# Doing so let the optimizer treat a sort on `l.v` as also sorting `r.v` and+# drop the `ORDER BY r.v` sort, returning wrongly ordered rows.+statement ok+CREATE TABLE eq_t1 (v INT);++statement ok+CREATE TABLE eq_t2 (v INT);++statement ok+INSERT INTO eq_t1 VALUES (1), (2), (3), (5), (8);++statement ok+INSERT INTO eq_t2 VALUES (4), (6), (9), (2);++query II+SELECT t1.v AS lv, t2.v AS rv+FROM eq_t1 t1+JOIN eq_t2 t2+ ON t1.v < t2.v+WHERE t1.v = 2+ORDER BY t2.v;+----+2 4+2 6+2 9+# ------------------------------------------------------------------# Existence joins (LeftSemi / LeftAnti) via PiecewiseMergeJoin# ------------------------------------------------------------------