Apache DataFusion SQL Query Engine
fix: preserve join predicates during reordering (#27066)
Phoenix5191bcc46d7e
diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rsindex 9cb624b80c..9e362ab09d 100644--- a/datafusion/optimizer/src/eliminate_cross_join.rs+++ b/datafusion/optimizer/src/eliminate_cross_join.rs@@ -14,7 +14,7 @@ use datafusion_expr::logical_plan::{Filter, Join, JoinConstraint, JoinType, LogicalPlan, Projection,};use datafusion_expr::utils::{can_hash, find_valid_equijoin_key_pair};-use datafusion_expr::{ExprSchemable, Operator, and, build_join_schema};+use datafusion_expr::{ExprSchemable, Operator, and, binary_expr, build_join_schema};/// Optimizer rule that uses filter predicates to replace cross joins with/// inner joins when join keys can be found.@@ -83,9 +83,9 @@ impl OptimizerRule for EliminateCrossJoin {let plan_schema = Arc::clone(plan.schema());let mut possible_join_keys = JoinKeySet::new();+ let mut null_equal_join_keys = JoinKeySet::new();let mut all_inputs: Vec<LogicalPlan> = vec![];let mut all_filters: Vec<Expr> = vec![];- let mut null_equality = NullEquality::NullEqualsNothing;let parent_predicate = if let LogicalPlan::Filter(filter) = plan {// if input isn't a join that can potentially be rewritten@@ -107,14 +107,10 @@ impl OptimizerRule for EliminateCrossJoin {input, predicate, ..} = filter;- // Extract null_equality setting from the input join- if let LogicalPlan::Join(join) = input.as_ref() {- null_equality = join.null_equality;- }-flatten_join_inputs(Arc::unwrap_or_clone(input),&mut possible_join_keys,+ &mut null_equal_join_keys,&mut all_inputs,&mut all_filters,);@@ -125,16 +121,15 @@ impl OptimizerRule for EliminateCrossJoin {match plan {LogicalPlan::Join(Join {join_type: JoinType::Inner,- null_equality: original_null_equality,..}) => {flatten_join_inputs(plan,&mut possible_join_keys,+ &mut null_equal_join_keys,&mut all_inputs,&mut all_filters,);- null_equality = original_null_equality;None}_ => {@@ -156,14 +151,16 @@ impl OptimizerRule for EliminateCrossJoin {// Join keys are handled locally:let mut all_join_keys = JoinKeySet::new();+ let mut all_null_equal_join_keys = JoinKeySet::new();let mut left = all_inputs.remove(0);while !all_inputs.is_empty() {left = find_inner_join(left,&mut all_inputs,&possible_join_keys,+ &null_equal_join_keys,&mut all_join_keys,- null_equality,+ &mut all_null_equal_join_keys,)?// Join keys may contain uncorrelated subqueries to rewrite.map_uncorrelated_subqueries(|input| self.rewrite(input, config))?@@ -177,29 +174,22 @@ impl OptimizerRule for EliminateCrossJoin {));}- if !all_filters.is_empty() {- // Add any filters on top - PushDownFilter can push filters down to applicable join- let first = all_filters.swap_remove(0);- let predicate = all_filters.into_iter().fold(first, and);- left = LogicalPlan::Filter(Filter::try_new(predicate, Arc::new(left))?);+ if let Some(predicate) = parent_predicate {+ all_filters.push(predicate);}- let Some(predicate) = parent_predicate else {- return Ok(Transformed::yes(left));- };-- // If there are no join keys then do nothing:- if all_join_keys.is_empty() {- Filter::try_new(predicate, Arc::new(left))- .map(|filter| Transformed::yes(LogicalPlan::Filter(filter)))- } else {- // Remove join expressions from filter:- match remove_join_expressions(predicate, &all_join_keys) {- Some(filter_expr) => Filter::try_new(filter_expr, Arc::new(left))- .map(|filter| Transformed::yes(LogicalPlan::Filter(filter))),- _ => Ok(Transformed::yes(left)),- }+ // Only remove predicates enforced by rebuilt joins with the same NULL+ // semantics. Keys spanning several inputs may need to remain as filters.+ let predicate = all_filters+ .into_iter()+ .filter_map(|expr| {+ remove_join_expressions(expr, &all_join_keys, &all_null_equal_join_keys)+ })+ .reduce(and);+ if let Some(predicate) = predicate {+ left = LogicalPlan::Filter(Filter::try_new(predicate, Arc::new(left))?);}+ Ok(Transformed::yes(left))}fn name(&self) -> &str {@@ -261,6 +251,7 @@ fn rewrite_children(fn flatten_join_inputs(plan: LogicalPlan,possible_join_keys: &mut JoinKeySet,+ null_equal_join_keys: &mut JoinKeySet,all_inputs: &mut Vec<LogicalPlan>,all_filters: &mut Vec<Expr>,) {@@ -269,16 +260,32 @@ fn flatten_join_inputs(if let Some(filter) = join.filter {all_filters.push(filter);}- possible_join_keys.insert_all_owned(join.on);+ // Preserve the full predicate until a rebuilt join enforces it.+ let (join_keys, op) = match join.null_equality {+ NullEquality::NullEqualsNothing => {+ (&mut *possible_join_keys, Operator::Eq)+ }+ NullEquality::NullEqualsNull => {+ (&mut *null_equal_join_keys, Operator::IsNotDistinctFrom)+ }+ };+ all_filters.extend(+ join.on+ .iter()+ .map(|(left, right)| binary_expr(left.clone(), op, right.clone())),+ );+ join_keys.insert_all_owned(join.on);flatten_join_inputs(Arc::unwrap_or_clone(join.left),possible_join_keys,+ null_equal_join_keys,all_inputs,all_filters,);flatten_join_inputs(Arc::unwrap_or_clone(join.right),possible_join_keys,+ null_equal_join_keys,all_inputs,all_filters,);@@ -305,31 +312,53 @@ fn find_inner_join(left_input: LogicalPlan,rights: &mut Vec<LogicalPlan>,possible_join_keys: &JoinKeySet,+ null_equal_join_keys: &JoinKeySet,all_join_keys: &mut JoinKeySet,- null_equality: NullEquality,+ all_null_equal_join_keys: &mut JoinKeySet,) -> Result<LogicalPlan> {for (i, right_input) in rights.iter().enumerate() {let mut join_keys = vec![];+ let mut null_equality = NullEquality::NullEqualsNothing;- for (l, r) in possible_join_keys.iter() {- let key_pair = find_valid_equijoin_key_pair(- l,- r,- left_input.schema(),- right_input.schema(),- )?;+ // A Join has one NULL equality setting for all of its keys. Any keys+ // with different semantics remain as residual predicates.+ for (possible_join_keys, all_join_keys, key_null_equality) in [+ (+ possible_join_keys,+ &mut *all_join_keys,+ NullEquality::NullEqualsNothing,+ ),+ (+ null_equal_join_keys,+ &mut *all_null_equal_join_keys,+ NullEquality::NullEqualsNull,+ ),+ ] {+ for (l, r) in possible_join_keys.iter() {+ let key_pair = find_valid_equijoin_key_pair(+ l,+ r,+ left_input.schema(),+ right_input.schema(),+ )?;++ // Save join keys+ if let Some((valid_l, valid_r)) = key_pair+ && can_hash(&valid_l.get_type(left_input.schema())?)+ {+ join_keys.push((valid_l, valid_r));+ }+ }- // Save join keys- if let Some((valid_l, valid_r)) = key_pair- && can_hash(&valid_l.get_type(left_input.schema())?)- {- join_keys.push((valid_l, valid_r));+ if !join_keys.is_empty() {+ all_join_keys.insert_all(join_keys.iter());+ null_equality = key_null_equality;+ break;}}// Found one or more matching join keysif !join_keys.is_empty() {- all_join_keys.insert_all(join_keys.iter());let right_input = rights.remove(i);let join_schema = Arc::new(build_join_schema(left_input.schema(),@@ -369,7 +398,7 @@ fn find_inner_join(filter: None,join_type: JoinType::Inner,join_constraint: JoinConstraint::On,- null_equality,+ null_equality: NullEquality::NullEqualsNothing,null_aware: false,null_aware_value_keys: 0,}))@@ -407,7 +436,11 @@ fn extract_possible_join_keys(expr: &Expr, join_keys: &mut JoinKeySet) {/// # Returns/// * `Some()` when there are few remaining predicates in filter_expr/// * `None` otherwise-fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {+fn remove_join_expressions(+ expr: Expr,+ join_keys: &JoinKeySet,+ null_equal_join_keys: &JoinKeySet,+) -> Option<Expr> {match expr {Expr::BinaryExpr(BinaryExpr {left,@@ -417,10 +450,15 @@ fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {// was a join key, so remove itNone}+ Expr::BinaryExpr(BinaryExpr {+ left,+ op: Operator::IsNotDistinctFrom,+ right,+ }) if null_equal_join_keys.contains(&left, &right) => None,// Fix for issue#78 join predicates from inside of OR expr also pulled up properly.Expr::BinaryExpr(BinaryExpr { left, op, right }) if op == Operator::And => {- let l = remove_join_expressions(*left, join_keys);- let r = remove_join_expressions(*right, join_keys);+ let l = remove_join_expressions(*left, join_keys, null_equal_join_keys);+ let r = remove_join_expressions(*right, join_keys, null_equal_join_keys);match (l, r) {(Some(ll), Some(rr)) => Some(Expr::BinaryExpr(BinaryExpr::new(Box::new(ll),@@ -433,8 +471,8 @@ fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {}}Expr::BinaryExpr(BinaryExpr { left, op, right }) if op == Operator::Or => {- let l = remove_join_expressions(*left, join_keys);- let r = remove_join_expressions(*right, join_keys);+ let l = remove_join_expressions(*left, join_keys, null_equal_join_keys);+ let r = remove_join_expressions(*right, join_keys, null_equal_join_keys);match (l, r) {(Some(ll), Some(rr)) => Some(Expr::BinaryExpr(BinaryExpr::new(Box::new(ll),@@ -462,6 +500,10 @@ mod tests {logical_plan::builder::LogicalPlanBuilder,scalar_subquery,};++ use NullEquality::{NullEqualsNothing, NullEqualsNull};+ use arrow::datatypes::Schema;+ use datafusion_expr::logical_plan::table_scan;use insta::assert_snapshot;macro_rules! assert_optimized_plan_equal {@@ -672,13 +714,12 @@ mod tests {assert_optimized_plan_equal!(plan,@ r"- Filter: t1.a > UInt32(15) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]- Filter: t1.a > UInt32(20) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]- Inner Join: t1.a = t2.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]- Inner Join: t1.a = t3.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]- TableScan: t1 [a:UInt32, b:UInt32, c:UInt32]- TableScan: t3 [a:UInt32, b:UInt32, c:UInt32]- TableScan: t2 [a:UInt32, b:UInt32, c:UInt32]+ Filter: t1.a > UInt32(20) AND t1.a > UInt32(15) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]+ Inner Join: t1.a = t2.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]+ Inner Join: t1.a = t3.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]+ TableScan: t1 [a:UInt32, b:UInt32, c:UInt32]+ TableScan: t3 [a:UInt32, b:UInt32, c:UInt32]+ TableScan: t2 [a:UInt32, b:UInt32, c:UInt32]")}@@ -1424,69 +1465,95 @@ mod tests {}#[test]- fn preserve_null_equality_setting() -> Result<()> {- let t1 = test_table_scan_with_name("t1")?;- let t2 = test_table_scan_with_name("t2")?;-- // Create an inner join with NullEquality::NullEqualsNull- let join_schema = Arc::new(build_join_schema(- t1.schema(),- t2.schema(),- &JoinType::Inner,- )?);-- let inner_join = LogicalPlan::Join(Join {- left: Arc::new(t1),- right: Arc::new(t2),- join_type: JoinType::Inner,- join_constraint: JoinConstraint::On,- on: vec![],- filter: None,- schema: join_schema,- null_equality: NullEquality::NullEqualsNull, // Test preservation- null_aware: false,- null_aware_value_keys: 0,- });-- // Apply filter that can create join conditions- let plan = LogicalPlanBuilder::from(inner_join)- .filter(binary_expr(- col("t1.a").eq(col("t2.a")),- And,- col("t2.c").lt(lit(20u32)),- ))?- .build()?;-- let rule = EliminateCrossJoin::new();- let optimized_plan = rule.rewrite(plan, &OptimizerContext::new())?.data;-- // Verify that null_equality is preserved in the optimized plan- fn check_null_equality_preserved(plan: &LogicalPlan) -> bool {- match plan {- LogicalPlan::Join(join) => {- // All joins in the optimized plan should preserve null equality- if join.null_equality == NullEquality::NullEqualsNothing {- return false;- }- // Recursively check child plans- plan.inputs()- .iter()- .all(|input| check_null_equality_preserved(input))- }- _ => {- // Recursively check child plans for non-join nodes- plan.inputs()- .iter()- .all(|input| check_null_equality_preserved(input))- }+ fn preserve_join_key_null_semantics() -> Result<()> {+ let schema = Schema::new(+ test_table_scan_fields()+ .into_iter()+ .map(|field| field.with_nullable(true))+ .collect::<Vec<_>>(),+ );+ let scan = |name| table_scan(Some(name), &schema, None)?.build();++ let test_scenarios = [+ // A filter's ordinary equality must not inherit an empty join's NULL setting.+ ("filter_key", None, None, NullEqualsNothing),+ // An existing NULL-safe key keeps its semantics when the join is rebuilt.+ (+ "existing_null_safe_key",+ Some(NullEqualsNull),+ None,+ NullEqualsNull,+ ),+ // An enclosing ordinary join must not override the inner NULL-safe key.+ (+ "null_safe_key_under_ordinary_join",+ Some(NullEqualsNull),+ Some(NullEqualsNothing),+ NullEqualsNull,+ ),+ // An enclosing NULL-safe join must not override the inner ordinary key.+ (+ "ordinary_key_under_null_safe_join",+ Some(NullEqualsNothing),+ Some(NullEqualsNull),+ NullEqualsNothing,+ ),+ ];+ for (name, existing_null_equality, outer_null_equality, expected_null_equality) in+ test_scenarios+ {+ let keys = if existing_null_equality.is_some() {+ (vec!["t1.a"], vec!["t2.a"])+ } else {+ (vec![], vec![])+ };+ let mut plan = LogicalPlanBuilder::from(scan("t1")?).join_detailed(+ scan("t2")?,+ JoinType::Inner,+ keys,+ None,+ existing_null_equality.unwrap_or(NullEqualsNull),+ )?;+ if let Some(null_equality) = outer_null_equality {+ plan = plan.join_detailed(+ scan("t3")?,+ JoinType::Inner,+ (vec!["t1.b"], vec!["t3.b"]),+ None,+ null_equality,+ )?;+ }+ let residual = col("t2.c").lt(lit(20u32));+ let predicate = if existing_null_equality.is_none() {+ col("t1.a").eq(col("t2.a")).and(residual.clone())+ } else {+ residual.clone()+ };+ let plan = plan.filter(predicate)?.build()?;+ let optimized_plan = EliminateCrossJoin::new()+ .rewrite(plan, &OptimizerContext::new())?+ .data;++ let LogicalPlan::Filter(filter) = optimized_plan else {+ panic!("{name}: expected residual filter");+ };+ assert_eq!(filter.predicate, residual, "{name}");+ let mut input = filter.input.as_ref();+ if let Some(null_equality) = outer_null_equality {+ let LogicalPlan::Join(join) = input else {+ panic!("{name}: expected outer join");+ };+ assert_eq!(join.on, vec![(col("t1.b"), col("t3.b"))], "{name}");+ assert_eq!(join.null_equality, null_equality, "{name}");+ input = join.left.as_ref();}+ let LogicalPlan::Join(join) = input else {+ panic!("{name}: expected inner join");+ };+ assert_eq!(join.on, vec![(col("t1.a"), col("t2.a"))], "{name}");+ assert_eq!(join.null_equality, expected_null_equality, "{name}");}- assert!(- check_null_equality_preserved(&optimized_plan),- "null_equality setting should be preserved after optimization"- );-Ok(())}diff --git a/datafusion/optimizer/src/join_key_set.rs b/datafusion/optimizer/src/join_key_set.rsindex 5d09651e40..515277d2b7 100644--- a/datafusion/optimizer/src/join_key_set.rs+++ b/datafusion/optimizer/src/join_key_set.rs@@ -105,6 +105,7 @@ impl JoinKeySet {}/// returns true if this set is empty+ #[cfg(test)]pub fn is_empty(&self) -> bool {self.inner.is_empty()}diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex d2be4f6900..db9d138849 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -2061,6 +2061,61 @@ set datafusion.explain.logical_plan_only = false;statement okset datafusion.execution.target_partitions = 4;+# A join key spanning two cross-joined inputs must survive join reordering.+query II+SELECT k.k, b.id+FROM (VALUES (1, 17, 18)) AS a(t, x, y)+CROSS JOIN (VALUES (0), (1)) AS k(k)+JOIN (VALUES (1, 17), (1, 18), (1, 19)) AS b(t, id)+ ON a.t = b.t AND b.id = CASE k.k WHEN 0 THEN a.x ELSE a.y END+ORDER BY k.k, b.id;+----+0 17+1 18++# A null-safe child join must keep its NULL matches under an ordinary join.+query I+SELECT a.id+FROM (VALUES (1, CAST(NULL AS INT)), (2, 5)) AS a(id, x)+JOIN (VALUES (CAST(NULL AS INT)), (5)) AS k(x)+ ON a.x IS NOT DISTINCT FROM k.x+JOIN (VALUES (1), (2)) AS b(id) ON a.id = b.id+ORDER BY a.id;+----+1+2++# An ordinary child join must not gain NULL matches under a null-safe join.+query I+SELECT a.id+FROM (VALUES (1, CAST(NULL AS INT)), (2, 5)) AS a(id, x)+JOIN (VALUES (CAST(NULL AS INT)), (5)) AS k(x) ON a.x = k.x+JOIN (VALUES (1), (2)) AS b(id) ON a.id IS NOT DISTINCT FROM b.id+ORDER BY a.id;+----+2++# A residual join key must retain its null-safe comparison when used as a filter.+query I+SELECT b.x+FROM (VALUES (1, CAST(NULL AS INT))) AS a(id, x)+CROSS JOIN (VALUES (0)) AS k(x)+JOIN (VALUES (1, CAST(NULL AS INT)), (1, 7)) AS b(id, x)+ ON a.id IS NOT DISTINCT FROM b.id+ AND a.x + k.x IS NOT DISTINCT FROM b.x;+----+NULL++# Using a null-safe key does not enforce an ordinary equality on the same columns.+query I+SELECT a.x+FROM (VALUES (CAST(NULL AS INT)), (1)) AS a(x)+JOIN (VALUES (CAST(NULL AS INT)), (1)) AS b(x)+ ON a.x IS NOT DISTINCT FROM b.x+WHERE a.x = b.x;+----+1+# Planning inner nested loop join# inputs are swapped due to inexact statistics + join reordering caused additional projection