Apache DataFusion SQL Query Engine
fix: keep probe-side constants in join equivalence properties (#26459)
Jay Zhanbd219dc192b9
diff --git a/datafusion/physical-expr/src/equivalence/class.rs b/datafusion/physical-expr/src/equivalence/class.rsindex b68d20fccf..ff34190bb1 100644--- a/datafusion/physical-expr/src/equivalence/class.rs+++ b/datafusion/physical-expr/src/equivalence/class.rs@@ -245,7 +245,8 @@ impl EquivalenceClass {}/// Adds the given offset to all columns in the expressions inside this- /// class. This is used when schemas are appended, e.g. in joins.+ /// class, keeping its constant value. This is used when schemas are+ /// appended, e.g. in joins.pub fn try_with_offset(&self, offset: isize) -> Result<Self> {let mut cls = Self::default();for expr_result in self@@ -256,6 +257,7 @@ impl EquivalenceClass {{cls.push(expr_result?);}+ cls.constant.clone_from(&self.constant);Ok(cls)}}@@ -823,7 +825,16 @@ impl EquivalenceGroup {self.iter().cloned().chain(right_equivalences.iter()- .map(|cls| cls.try_with_offset(left_size as _))+ .map(|cls| {+ let mut cls = cls.try_with_offset(left_size as _)?;+ // Rows without a match on the right side are+ // padded with NULLs, so its constants no+ // longer hold.+ if matches!(join_type, JoinType::Left | JoinType::Full) {+ cls.constant = None;+ }+ Ok(cls)+ }).collect::<Result<Vec<_>>>()?,),);@@ -1351,6 +1362,34 @@ mod tests {Ok(())}+ /// A constant on the right side of a join stays constant in the join+ /// output, unless unmatched rows pad the right side with NULLs.+ /// <https://github.com/apache/datafusion/issues/26283>+ #[test]+ fn test_join_keeps_right_constants() -> Result<()> {+ let schema = abc_schema();+ let mut right = EquivalenceGroup::default();+ right.add_constant(ConstExpr::from(col("a", &schema)?));+ // `a` of the right side is column 4 of the join output+ let right_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 4));++ for (join_type, expected) in [+ (JoinType::Inner, true),+ (JoinType::Right, true),+ (JoinType::Left, false),+ (JoinType::Full, false),+ ] {+ let joined = EquivalenceGroup::default().join(&right, &join_type, 4, &[])?;+ assert_eq!(+ joined.is_expr_constant(&right_a).is_some(),+ expected,+ "{join_type}"+ );+ }++ Ok(())+ }+#[test]fn test_has_same_classes_ignores_the_map() -> Result<()> {// `map` indexes into `classes`, so equal classes must imply equaldiff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex df8a3a8aad..03e5c740a9 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -149,13 +149,15 @@ impl CrossJoinExec {right: &Arc<dyn ExecutionPlan>,schema: SchemaRef,) -> Result<PlanProperties> {- // Calculate equivalence properties+ // Calculate equivalence properties. A cross join is an inner join+ // without keys: no side is padded with NULLs, so the constants of+ // both sides still hold.// TODO: Check equivalence properties of cross join, it may preserve// ordering in some cases.let eq_properties = join_equivalence_properties(left.equivalence_properties().clone(),right.equivalence_properties().clone(),- &JoinType::Full,+ &JoinType::Inner,schema,&[false, false],None,diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 0d48421a83..3514c30e3c 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -2333,3 +2333,136 @@ DROP TABLE spm_orderings;statement okset datafusion.execution.target_partitions = 4;++# `k` is constant on the probe side of the join, so the join output reports+# it as constant and no sort on `k` is needed.+# https://github.com/apache/datafusion/issues/26283+statement ok+CREATE TABLE probe_constant_t(a INT, b INT) AS VALUES (1, 1), (2, 2), (3, 3), (4, 4);++statement ok+CREATE TABLE probe_constant_u(a INT) AS VALUES (1);++statement ok+CREATE TABLE probe_constant_v(x INT) AS VALUES (1), (2);++query II+SELECT s.k, s.a+FROM (+ SELECT probe_constant_t.a, -2 AS k+ FROM probe_constant_t JOIN probe_constant_v ON probe_constant_t.b <> probe_constant_v.x+) s+JOIN probe_constant_u ON s.a = probe_constant_u.a+ORDER BY s.k DESC;+----+-2 1++query TT+EXPLAIN SELECT s.k, s.a+FROM (+ SELECT probe_constant_t.a, -2 AS k+ FROM probe_constant_t JOIN probe_constant_v ON probe_constant_t.b <> probe_constant_v.x+) s+JOIN probe_constant_u ON s.a = probe_constant_u.a+ORDER BY s.k DESC;+----+logical_plan+01)Sort: s.k DESC NULLS FIRST+02)--Projection: s.k, s.a+03)----Inner Join: s.a = probe_constant_u.a+04)------SubqueryAlias: s+05)--------Projection: probe_constant_t.a, Int64(-2) AS k+06)----------Inner Join: Filter: probe_constant_t.b != probe_constant_v.x+07)------------TableScan: probe_constant_t projection=[a, b]+08)------------TableScan: probe_constant_v projection=[x]+09)------TableScan: probe_constant_u projection=[a]+physical_plan+01)CoalescePartitionsExec+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)], projection=[k@2, a@1]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)------ProjectionExec: expr=[a@0 as a, -2 as k]+06)--------NestedLoopJoinExec: join_type=Inner, filter=b@0 != x@1, projection=[a@1]+07)----------DataSourceExec: partitions=1, partition_sizes=[1]+08)----------DataSourceExec: partitions=1, partition_sizes=[1]++statement ok+SET datafusion.execution.target_partitions = 1;++query ITI+SELECT s.a, s.k, row_number() OVER (PARTITION BY s.a ORDER BY s.k) AS r+FROM probe_constant_u JOIN (SELECT a, 'x' AS k FROM probe_constant_t) AS s+ ON s.a = probe_constant_u.a;+----+1 x 1++query TT+EXPLAIN SELECT s.a, s.k, row_number() OVER (PARTITION BY s.a ORDER BY s.k) AS r+FROM probe_constant_u JOIN (SELECT a, 'x' AS k FROM probe_constant_t) AS s+ ON s.a = probe_constant_u.a;+----+logical_plan+01)Projection: s.a, s.k, row_number() PARTITION BY [s.a] ORDER BY [s.k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS r+02)--WindowAggr: windowExpr=[[row_number() PARTITION BY [s.a] ORDER BY [s.k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]+03)----Projection: s.a, s.k+04)------Inner Join: probe_constant_u.a = s.a+05)--------TableScan: probe_constant_u projection=[a]+06)--------SubqueryAlias: s+07)----------Projection: probe_constant_t.a, Utf8("x") AS k+08)------------TableScan: probe_constant_t projection=[a]+physical_plan+01)ProjectionExec: expr=[a@0 as a, k@1 as k, row_number() PARTITION BY [s.a] ORDER BY [s.k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [s.a] ORDER BY [s.k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [s.a] ORDER BY [s.k ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)], projection=[a@1, k@2]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)------SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+06)--------FilterExec: DynamicFilter [ empty ]+07)----------ProjectionExec: expr=[a@0 as a, x as k]+08)------------DataSourceExec: partitions=1, partition_sizes=[1]++# Unmatched rows of a left join pad the probe side with NULLs, so `k` is not+# constant in the join output and the sort on `k` is kept.+query TT+EXPLAIN SELECT u.a, s.k+FROM (VALUES (1), (10)) AS u(a) LEFT JOIN (SELECT a, 'x' AS k FROM probe_constant_t) AS s+ ON s.a = u.a+ORDER BY s.k DESC NULLS FIRST, u.a;+----+logical_plan+01)Sort: s.k DESC NULLS FIRST, u.a ASC NULLS LAST+02)--Projection: u.a, s.k+03)----Left Join: u.a = CAST(s.a AS Int64)+04)------SubqueryAlias: u+05)--------Projection: column1 AS a+06)----------Values: (Int64(1)), (Int64(10))+07)------SubqueryAlias: s+08)--------Projection: probe_constant_t.a, Utf8("x") AS k+09)----------TableScan: probe_constant_t projection=[a]+physical_plan+01)SortExec: expr=[k@1 DESC, a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, CAST(s.a AS Int64)@2)], projection=[a@0, k@2]+03)----ProjectionExec: expr=[column1@0 as a]+04)------DataSourceExec: partitions=1, partition_sizes=[1]+05)----ProjectionExec: expr=[a@0 as a, x as k, CAST(a@0 AS Int64) as CAST(s.a AS Int64)]+06)------DataSourceExec: partitions=1, partition_sizes=[1]++query IT+SELECT u.a, s.k+FROM (VALUES (1), (10)) AS u(a) LEFT JOIN (SELECT a, 'x' AS k FROM probe_constant_t) AS s+ ON s.a = u.a+ORDER BY s.k DESC NULLS FIRST, u.a;+----+10 NULL+1 x++statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+DROP TABLE probe_constant_t;++statement ok+DROP TABLE probe_constant_u;++statement ok+DROP TABLE probe_constant_v;