Skip to content

Apache DataFusion SQL Query Engine

fix: preserve join predicates during reordering (#27066)

Phoenix5191bcc46d7e
diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rs
index 9cb624b80c..9e362ab09d 100644
--- a/datafusion/optimizer/src/eliminate_cross_join.rs
+++ b/datafusion/optimizer/src/eliminate_cross_join.rs
@@ -14,7 +14,7 @@ use datafusion_expr::logical_plan::{
Filter, Join, JoinConstraint, JoinType, LogicalPlan, Projection,
};
use datafusion_expr::utils::{can_hash, find_valid_equijoin_key_pair};
-use datafusion_expr::{ExprSchemable, Operator, and, build_join_schema};
+use datafusion_expr::{ExprSchemable, Operator, and, binary_expr, build_join_schema};
/// Optimizer rule that uses filter predicates to replace cross joins with
/// inner joins when join keys can be found.
@@ -83,9 +83,9 @@ impl OptimizerRule for EliminateCrossJoin {
let plan_schema = Arc::clone(plan.schema());
let mut possible_join_keys = JoinKeySet::new();
+ let mut null_equal_join_keys = JoinKeySet::new();
let mut all_inputs: Vec<LogicalPlan> = vec![];
let mut all_filters: Vec<Expr> = vec![];
- let mut null_equality = NullEquality::NullEqualsNothing;
let parent_predicate = if let LogicalPlan::Filter(filter) = plan {
// if input isn't a join that can potentially be rewritten
@@ -107,14 +107,10 @@ impl OptimizerRule for EliminateCrossJoin {
input, predicate, ..
} = filter;
- // Extract null_equality setting from the input join
- if let LogicalPlan::Join(join) = input.as_ref() {
- null_equality = join.null_equality;
- }
-
flatten_join_inputs(
Arc::unwrap_or_clone(input),
&mut possible_join_keys,
+ &mut null_equal_join_keys,
&mut all_inputs,
&mut all_filters,
);
@@ -125,16 +121,15 @@ impl OptimizerRule for EliminateCrossJoin {
match plan {
LogicalPlan::Join(Join {
join_type: JoinType::Inner,
- null_equality: original_null_equality,
..
}) => {
flatten_join_inputs(
plan,
&mut possible_join_keys,
+ &mut null_equal_join_keys,
&mut all_inputs,
&mut all_filters,
);
- null_equality = original_null_equality;
None
}
_ => {
@@ -156,14 +151,16 @@ impl OptimizerRule for EliminateCrossJoin {
// Join keys are handled locally:
let mut all_join_keys = JoinKeySet::new();
+ let mut all_null_equal_join_keys = JoinKeySet::new();
let mut left = all_inputs.remove(0);
while !all_inputs.is_empty() {
left = find_inner_join(
left,
&mut all_inputs,
&possible_join_keys,
+ &null_equal_join_keys,
&mut all_join_keys,
- null_equality,
+ &mut all_null_equal_join_keys,
)?
// Join keys may contain uncorrelated subqueries to rewrite
.map_uncorrelated_subqueries(|input| self.rewrite(input, config))?
@@ -177,29 +174,22 @@ impl OptimizerRule for EliminateCrossJoin {
));
}
- if !all_filters.is_empty() {
- // Add any filters on top - PushDownFilter can push filters down to applicable join
- let first = all_filters.swap_remove(0);
- let predicate = all_filters.into_iter().fold(first, and);
- left = LogicalPlan::Filter(Filter::try_new(predicate, Arc::new(left))?);
+ if let Some(predicate) = parent_predicate {
+ all_filters.push(predicate);
}
- let Some(predicate) = parent_predicate else {
- return Ok(Transformed::yes(left));
- };
-
- // If there are no join keys then do nothing:
- if all_join_keys.is_empty() {
- Filter::try_new(predicate, Arc::new(left))
- .map(|filter| Transformed::yes(LogicalPlan::Filter(filter)))
- } else {
- // Remove join expressions from filter:
- match remove_join_expressions(predicate, &all_join_keys) {
- Some(filter_expr) => Filter::try_new(filter_expr, Arc::new(left))
- .map(|filter| Transformed::yes(LogicalPlan::Filter(filter))),
- _ => Ok(Transformed::yes(left)),
- }
+ // Only remove predicates enforced by rebuilt joins with the same NULL
+ // semantics. Keys spanning several inputs may need to remain as filters.
+ let predicate = all_filters
+ .into_iter()
+ .filter_map(|expr| {
+ remove_join_expressions(expr, &all_join_keys, &all_null_equal_join_keys)
+ })
+ .reduce(and);
+ if let Some(predicate) = predicate {
+ left = LogicalPlan::Filter(Filter::try_new(predicate, Arc::new(left))?);
}
+ Ok(Transformed::yes(left))
}
fn name(&self) -> &str {
@@ -261,6 +251,7 @@ fn rewrite_children(
fn flatten_join_inputs(
plan: LogicalPlan,
possible_join_keys: &mut JoinKeySet,
+ null_equal_join_keys: &mut JoinKeySet,
all_inputs: &mut Vec<LogicalPlan>,
all_filters: &mut Vec<Expr>,
) {
@@ -269,16 +260,32 @@ fn flatten_join_inputs(
if let Some(filter) = join.filter {
all_filters.push(filter);
}
- possible_join_keys.insert_all_owned(join.on);
+ // Preserve the full predicate until a rebuilt join enforces it.
+ let (join_keys, op) = match join.null_equality {
+ NullEquality::NullEqualsNothing => {
+ (&mut *possible_join_keys, Operator::Eq)
+ }
+ NullEquality::NullEqualsNull => {
+ (&mut *null_equal_join_keys, Operator::IsNotDistinctFrom)
+ }
+ };
+ all_filters.extend(
+ join.on
+ .iter()
+ .map(|(left, right)| binary_expr(left.clone(), op, right.clone())),
+ );
+ join_keys.insert_all_owned(join.on);
flatten_join_inputs(
Arc::unwrap_or_clone(join.left),
possible_join_keys,
+ null_equal_join_keys,
all_inputs,
all_filters,
);
flatten_join_inputs(
Arc::unwrap_or_clone(join.right),
possible_join_keys,
+ null_equal_join_keys,
all_inputs,
all_filters,
);
@@ -305,31 +312,53 @@ fn find_inner_join(
left_input: LogicalPlan,
rights: &mut Vec<LogicalPlan>,
possible_join_keys: &JoinKeySet,
+ null_equal_join_keys: &JoinKeySet,
all_join_keys: &mut JoinKeySet,
- null_equality: NullEquality,
+ all_null_equal_join_keys: &mut JoinKeySet,
) -> Result<LogicalPlan> {
for (i, right_input) in rights.iter().enumerate() {
let mut join_keys = vec![];
+ let mut null_equality = NullEquality::NullEqualsNothing;
- for (l, r) in possible_join_keys.iter() {
- let key_pair = find_valid_equijoin_key_pair(
- l,
- r,
- left_input.schema(),
- right_input.schema(),
- )?;
+ // A Join has one NULL equality setting for all of its keys. Any keys
+ // with different semantics remain as residual predicates.
+ for (possible_join_keys, all_join_keys, key_null_equality) in [
+ (
+ possible_join_keys,
+ &mut *all_join_keys,
+ NullEquality::NullEqualsNothing,
+ ),
+ (
+ null_equal_join_keys,
+ &mut *all_null_equal_join_keys,
+ NullEquality::NullEqualsNull,
+ ),
+ ] {
+ for (l, r) in possible_join_keys.iter() {
+ let key_pair = find_valid_equijoin_key_pair(
+ l,
+ r,
+ left_input.schema(),
+ right_input.schema(),
+ )?;
+
+ // Save join keys
+ if let Some((valid_l, valid_r)) = key_pair
+ && can_hash(&valid_l.get_type(left_input.schema())?)
+ {
+ join_keys.push((valid_l, valid_r));
+ }
+ }
- // Save join keys
- if let Some((valid_l, valid_r)) = key_pair
- && can_hash(&valid_l.get_type(left_input.schema())?)
- {
- join_keys.push((valid_l, valid_r));
+ if !join_keys.is_empty() {
+ all_join_keys.insert_all(join_keys.iter());
+ null_equality = key_null_equality;
+ break;
}
}
// Found one or more matching join keys
if !join_keys.is_empty() {
- all_join_keys.insert_all(join_keys.iter());
let right_input = rights.remove(i);
let join_schema = Arc::new(build_join_schema(
left_input.schema(),
@@ -369,7 +398,7 @@ fn find_inner_join(
filter: None,
join_type: JoinType::Inner,
join_constraint: JoinConstraint::On,
- null_equality,
+ null_equality: NullEquality::NullEqualsNothing,
null_aware: false,
null_aware_value_keys: 0,
}))
@@ -407,7 +436,11 @@ fn extract_possible_join_keys(expr: &Expr, join_keys: &mut JoinKeySet) {
/// # Returns
/// * `Some()` when there are few remaining predicates in filter_expr
/// * `None` otherwise
-fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {
+fn remove_join_expressions(
+ expr: Expr,
+ join_keys: &JoinKeySet,
+ null_equal_join_keys: &JoinKeySet,
+) -> Option<Expr> {
match expr {
Expr::BinaryExpr(BinaryExpr {
left,
@@ -417,10 +450,15 @@ fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {
// was a join key, so remove it
None
}
+ Expr::BinaryExpr(BinaryExpr {
+ left,
+ op: Operator::IsNotDistinctFrom,
+ right,
+ }) if null_equal_join_keys.contains(&left, &right) => None,
// Fix for issue#78 join predicates from inside of OR expr also pulled up properly.
Expr::BinaryExpr(BinaryExpr { left, op, right }) if op == Operator::And => {
- let l = remove_join_expressions(*left, join_keys);
- let r = remove_join_expressions(*right, join_keys);
+ let l = remove_join_expressions(*left, join_keys, null_equal_join_keys);
+ let r = remove_join_expressions(*right, join_keys, null_equal_join_keys);
match (l, r) {
(Some(ll), Some(rr)) => Some(Expr::BinaryExpr(BinaryExpr::new(
Box::new(ll),
@@ -433,8 +471,8 @@ fn remove_join_expressions(expr: Expr, join_keys: &JoinKeySet) -> Option<Expr> {
}
}
Expr::BinaryExpr(BinaryExpr { left, op, right }) if op == Operator::Or => {
- let l = remove_join_expressions(*left, join_keys);
- let r = remove_join_expressions(*right, join_keys);
+ let l = remove_join_expressions(*left, join_keys, null_equal_join_keys);
+ let r = remove_join_expressions(*right, join_keys, null_equal_join_keys);
match (l, r) {
(Some(ll), Some(rr)) => Some(Expr::BinaryExpr(BinaryExpr::new(
Box::new(ll),
@@ -462,6 +500,10 @@ mod tests {
logical_plan::builder::LogicalPlanBuilder,
scalar_subquery,
};
+
+ use NullEquality::{NullEqualsNothing, NullEqualsNull};
+ use arrow::datatypes::Schema;
+ use datafusion_expr::logical_plan::table_scan;
use insta::assert_snapshot;
macro_rules! assert_optimized_plan_equal {
@@ -672,13 +714,12 @@ mod tests {
assert_optimized_plan_equal!(
plan,
@ r"
- Filter: t1.a > UInt32(15) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
- Filter: t1.a > UInt32(20) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
- Inner Join: t1.a = t2.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
- Inner Join: t1.a = t3.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
- TableScan: t1 [a:UInt32, b:UInt32, c:UInt32]
- TableScan: t3 [a:UInt32, b:UInt32, c:UInt32]
- TableScan: t2 [a:UInt32, b:UInt32, c:UInt32]
+ Filter: t1.a > UInt32(20) AND t1.a > UInt32(15) [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
+ Inner Join: t1.a = t2.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
+ Inner Join: t1.a = t3.a [a:UInt32, b:UInt32, c:UInt32, a:UInt32, b:UInt32, c:UInt32]
+ TableScan: t1 [a:UInt32, b:UInt32, c:UInt32]
+ TableScan: t3 [a:UInt32, b:UInt32, c:UInt32]
+ TableScan: t2 [a:UInt32, b:UInt32, c:UInt32]
"
)
}
@@ -1424,69 +1465,95 @@ mod tests {
}
#[test]
- fn preserve_null_equality_setting() -> Result<()> {
- let t1 = test_table_scan_with_name("t1")?;
- let t2 = test_table_scan_with_name("t2")?;
-
- // Create an inner join with NullEquality::NullEqualsNull
- let join_schema = Arc::new(build_join_schema(
- t1.schema(),
- t2.schema(),
- &JoinType::Inner,
- )?);
-
- let inner_join = LogicalPlan::Join(Join {
- left: Arc::new(t1),
- right: Arc::new(t2),
- join_type: JoinType::Inner,
- join_constraint: JoinConstraint::On,
- on: vec![],
- filter: None,
- schema: join_schema,
- null_equality: NullEquality::NullEqualsNull, // Test preservation
- null_aware: false,
- null_aware_value_keys: 0,
- });
-
- // Apply filter that can create join conditions
- let plan = LogicalPlanBuilder::from(inner_join)
- .filter(binary_expr(
- col("t1.a").eq(col("t2.a")),
- And,
- col("t2.c").lt(lit(20u32)),
- ))?
- .build()?;
-
- let rule = EliminateCrossJoin::new();
- let optimized_plan = rule.rewrite(plan, &OptimizerContext::new())?.data;
-
- // Verify that null_equality is preserved in the optimized plan
- fn check_null_equality_preserved(plan: &LogicalPlan) -> bool {
- match plan {
- LogicalPlan::Join(join) => {
- // All joins in the optimized plan should preserve null equality
- if join.null_equality == NullEquality::NullEqualsNothing {
- return false;
- }
- // Recursively check child plans
- plan.inputs()
- .iter()
- .all(|input| check_null_equality_preserved(input))
- }
- _ => {
- // Recursively check child plans for non-join nodes
- plan.inputs()
- .iter()
- .all(|input| check_null_equality_preserved(input))
- }
+ fn preserve_join_key_null_semantics() -> Result<()> {
+ let schema = Schema::new(
+ test_table_scan_fields()
+ .into_iter()
+ .map(|field| field.with_nullable(true))
+ .collect::<Vec<_>>(),
+ );
+ let scan = |name| table_scan(Some(name), &schema, None)?.build();
+
+ let test_scenarios = [
+ // A filter's ordinary equality must not inherit an empty join's NULL setting.
+ ("filter_key", None, None, NullEqualsNothing),
+ // An existing NULL-safe key keeps its semantics when the join is rebuilt.
+ (
+ "existing_null_safe_key",
+ Some(NullEqualsNull),
+ None,
+ NullEqualsNull,
+ ),
+ // An enclosing ordinary join must not override the inner NULL-safe key.
+ (
+ "null_safe_key_under_ordinary_join",
+ Some(NullEqualsNull),
+ Some(NullEqualsNothing),
+ NullEqualsNull,
+ ),
+ // An enclosing NULL-safe join must not override the inner ordinary key.
+ (
+ "ordinary_key_under_null_safe_join",
+ Some(NullEqualsNothing),
+ Some(NullEqualsNull),
+ NullEqualsNothing,
+ ),
+ ];
+ for (name, existing_null_equality, outer_null_equality, expected_null_equality) in
+ test_scenarios
+ {
+ let keys = if existing_null_equality.is_some() {
+ (vec!["t1.a"], vec!["t2.a"])
+ } else {
+ (vec![], vec![])
+ };
+ let mut plan = LogicalPlanBuilder::from(scan("t1")?).join_detailed(
+ scan("t2")?,
+ JoinType::Inner,
+ keys,
+ None,
+ existing_null_equality.unwrap_or(NullEqualsNull),
+ )?;
+ if let Some(null_equality) = outer_null_equality {
+ plan = plan.join_detailed(
+ scan("t3")?,
+ JoinType::Inner,
+ (vec!["t1.b"], vec!["t3.b"]),
+ None,
+ null_equality,
+ )?;
+ }
+ let residual = col("t2.c").lt(lit(20u32));
+ let predicate = if existing_null_equality.is_none() {
+ col("t1.a").eq(col("t2.a")).and(residual.clone())
+ } else {
+ residual.clone()
+ };
+ let plan = plan.filter(predicate)?.build()?;
+ let optimized_plan = EliminateCrossJoin::new()
+ .rewrite(plan, &OptimizerContext::new())?
+ .data;
+
+ let LogicalPlan::Filter(filter) = optimized_plan else {
+ panic!("{name}: expected residual filter");
+ };
+ assert_eq!(filter.predicate, residual, "{name}");
+ let mut input = filter.input.as_ref();
+ if let Some(null_equality) = outer_null_equality {
+ let LogicalPlan::Join(join) = input else {
+ panic!("{name}: expected outer join");
+ };
+ assert_eq!(join.on, vec![(col("t1.b"), col("t3.b"))], "{name}");
+ assert_eq!(join.null_equality, null_equality, "{name}");
+ input = join.left.as_ref();
}
+ let LogicalPlan::Join(join) = input else {
+ panic!("{name}: expected inner join");
+ };
+ assert_eq!(join.on, vec![(col("t1.a"), col("t2.a"))], "{name}");
+ assert_eq!(join.null_equality, expected_null_equality, "{name}");
}
- assert!(
- check_null_equality_preserved(&optimized_plan),
- "null_equality setting should be preserved after optimization"
- );
-
Ok(())
}
diff --git a/datafusion/optimizer/src/join_key_set.rs b/datafusion/optimizer/src/join_key_set.rs
index 5d09651e40..515277d2b7 100644
--- a/datafusion/optimizer/src/join_key_set.rs
+++ b/datafusion/optimizer/src/join_key_set.rs
@@ -105,6 +105,7 @@ impl JoinKeySet {
}
/// returns true if this set is empty
+ #[cfg(test)]
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index d2be4f6900..db9d138849 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -2061,6 +2061,61 @@ set datafusion.explain.logical_plan_only = false;
statement ok
set datafusion.execution.target_partitions = 4;
+# A join key spanning two cross-joined inputs must survive join reordering.
+query II
+SELECT k.k, b.id
+FROM (VALUES (1, 17, 18)) AS a(t, x, y)
+CROSS JOIN (VALUES (0), (1)) AS k(k)
+JOIN (VALUES (1, 17), (1, 18), (1, 19)) AS b(t, id)
+ ON a.t = b.t AND b.id = CASE k.k WHEN 0 THEN a.x ELSE a.y END
+ORDER BY k.k, b.id;
+----
+0 17
+1 18
+
+# A null-safe child join must keep its NULL matches under an ordinary join.
+query I
+SELECT a.id
+FROM (VALUES (1, CAST(NULL AS INT)), (2, 5)) AS a(id, x)
+JOIN (VALUES (CAST(NULL AS INT)), (5)) AS k(x)
+ ON a.x IS NOT DISTINCT FROM k.x
+JOIN (VALUES (1), (2)) AS b(id) ON a.id = b.id
+ORDER BY a.id;
+----
+1
+2
+
+# An ordinary child join must not gain NULL matches under a null-safe join.
+query I
+SELECT a.id
+FROM (VALUES (1, CAST(NULL AS INT)), (2, 5)) AS a(id, x)
+JOIN (VALUES (CAST(NULL AS INT)), (5)) AS k(x) ON a.x = k.x
+JOIN (VALUES (1), (2)) AS b(id) ON a.id IS NOT DISTINCT FROM b.id
+ORDER BY a.id;
+----
+2
+
+# A residual join key must retain its null-safe comparison when used as a filter.
+query I
+SELECT b.x
+FROM (VALUES (1, CAST(NULL AS INT))) AS a(id, x)
+CROSS JOIN (VALUES (0)) AS k(x)
+JOIN (VALUES (1, CAST(NULL AS INT)), (1, 7)) AS b(id, x)
+ ON a.id IS NOT DISTINCT FROM b.id
+ AND a.x + k.x IS NOT DISTINCT FROM b.x;
+----
+NULL
+
+# Using a null-safe key does not enforce an ordinary equality on the same columns.
+query I
+SELECT a.x
+FROM (VALUES (CAST(NULL AS INT)), (1)) AS a(x)
+JOIN (VALUES (CAST(NULL AS INT)), (1)) AS b(x)
+ ON a.x IS NOT DISTINCT FROM b.x
+WHERE a.x = b.x;
+----
+1
+
# Planning inner nested loop join
# inputs are swapped due to inexact statistics + join reordering caused additional projection