Skip to content

Apache DataFusion SQL Query Engine

Refactor outer join null-rejection analysis to track join sides directly (#22870)

kosiewddc157d10e89
diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rs
index 4691eaf48b..9ec99069f6 100644
--- a/datafusion/optimizer/src/eliminate_outer_join.rs
+++ b/datafusion/optimizer/src/eliminate_outer_join.rs
@@ -146,36 +146,27 @@ impl OptimizerRule for EliminateOuterJoin {
}
}
-/// Run the null-rejection analysis on `predicate` against `join`'s left/right
-/// schemas. Return `Some(new_join_plan)` if the join type can be tightened
-/// (e.g. LEFT → INNER), `None` otherwise.
+/// Attempt to simplify an outer join by analyzing `predicate` for
+/// null-rejection. If the predicate filters out rows padded with NULLs on one
+/// or both sides, return a copy of `join` rewritten to an equivalent join type
+/// that omits those rows in the first place; otherwise return `None`.
fn try_simplify_join(join: &Join, predicate: &Expr) -> Option<LogicalPlan> {
if !join.join_type.is_outer() {
return None;
}
- let mut null_rejecting_cols: Vec<Column> = vec![];
- extract_null_rejecting_columns(
+ let null_rejecting_sides = extract_null_rejecting_sides(
predicate,
- &mut null_rejecting_cols,
join.left.schema(),
join.right.schema(),
true,
);
- let mut left_non_nullable = false;
- let mut right_non_nullable = false;
- for col in null_rejecting_cols.iter() {
- if join.left.schema().has_column(col) {
- left_non_nullable = true;
- }
- if join.right.schema().has_column(col) {
- right_non_nullable = true;
- }
- }
-
- let new_join_type =
- eliminate_outer(join.join_type, left_non_nullable, right_non_nullable);
+ let new_join_type = eliminate_outer(
+ join.join_type,
+ null_rejecting_sides.left,
+ null_rejecting_sides.right,
+ );
if new_join_type == join.join_type {
return None;
}
@@ -252,190 +243,139 @@ pub fn eliminate_outer(
}
}
-/// Find the columns that `expr` rejects NULL on. If any of these columns are
-/// NULL, `expr` is guaranteed to evaluate to NULL or false, and the row
-/// therefore cannot survive a WHERE clause. Matching columns are appended to
-/// `null_rejecting_cols`.
-///
-/// The caller uses the result to decide whether an outer join's null-padded
-/// rows could survive the predicate above the join: if a column from the
-/// nullable side appears in `null_rejecting_cols`, it cannot, and the outer
-/// join can be converted to an inner join.
+#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)]
+struct NullRejectingSides {
+ left: bool,
+ right: bool,
+}
+
+impl NullRejectingSides {
+ /// The join side(s) a column belongs to.
+ ///
+ /// A bare column reference is null-rejecting on its own side: if the column
+ /// is NULL, every null-propagating operator above it yields NULL and the row
+ /// is filtered.
+ fn for_column(col: &Column, left_schema: &DFSchema, right_schema: &DFSchema) -> Self {
+ Self {
+ left: left_schema.has_column(col),
+ right: right_schema.has_column(col),
+ }
+ }
+
+ fn union(self, other: Self) -> Self {
+ Self {
+ left: self.left || other.left,
+ right: self.right || other.right,
+ }
+ }
+
+ fn intersection(self, other: Self) -> Self {
+ Self {
+ left: self.left && other.left,
+ right: self.right && other.right,
+ }
+ }
+}
+
+/// Compute which join sides are null-rejected by `expr` in a WHERE clause.
+/// For each marked side, rows padded with NULLs on that side are guaranteed to
+/// evaluate to NULL or false and be filtered out.
///
-/// `left_schema` and `right_schema` are the join's two child schemas.
-/// `top_level` is true at the root of the WHERE predicate and false on each
-/// recursion.
-fn extract_null_rejecting_columns(
+/// `left_schema` and `right_schema` map column references to join sides.
+/// `top_level` is true only while walking the root WHERE context; nested
+/// contexts are more conservative because their boolean result may be combined
+/// by an enclosing expression.
+fn extract_null_rejecting_sides(
expr: &Expr,
- null_rejecting_cols: &mut Vec<Column>,
left_schema: &Arc<DFSchema>,
right_schema: &Arc<DFSchema>,
top_level: bool,
-) {
+) -> NullRejectingSides {
match expr {
Expr::Column(col) => {
- null_rejecting_cols.push(col.clone());
+ NullRejectingSides::for_column(col, left_schema, right_schema)
}
Expr::BinaryExpr(BinaryExpr { left, op, right }) => match op {
Operator::And | Operator::Or => {
- // AND distributes only down a top-level AND chain in the WHERE
- // clause: each conjunct is independently null- rejecting, so
- // any column either side discovers is a column the WHERE
- // rejects NULL on. Once an AND appears below any other context,
- // we fall back to the per-side analysis used for OR, because
- // the context might influence whether the row is filtered.
- if top_level && *op == Operator::And {
- extract_null_rejecting_columns(
- left,
- null_rejecting_cols,
- left_schema,
- right_schema,
- top_level,
- );
- extract_null_rejecting_columns(
- right,
- null_rejecting_cols,
- left_schema,
- right_schema,
- top_level,
- );
- return;
- }
-
- // OR (and nested AND): a row survives if EITHER operand returns
- // true. We can credit a join side as null-rejecting only when
- // BOTH operands independently reject NULL on a column from that
- // side — otherwise the other branch could let the NULL row
- // through.
- let mut left_cols: Vec<Column> = vec![];
- let mut right_cols: Vec<Column> = vec![];
- extract_null_rejecting_columns(
+ let left_sides = extract_null_rejecting_sides(
left,
- &mut left_cols,
left_schema,
right_schema,
top_level,
);
- extract_null_rejecting_columns(
+ let right_sides = extract_null_rejecting_sides(
right,
- &mut right_cols,
left_schema,
right_schema,
top_level,
);
- let find_on = |cols: &[Column], schema: &DFSchema| {
- cols.iter().find(|c| schema.has_column(c)).cloned()
- };
- for schema in [left_schema, right_schema] {
- if let (Some(c), Some(_)) =
- (find_on(&left_cols, schema), find_on(&right_cols, schema))
- {
- null_rejecting_cols.push(c);
- }
+ // Top-level AND: each conjunct is an independent WHERE filter,
+ // so side evidence from either branch is sufficient.
+ // Nested AND is handled like OR because the enclosing context
+ // may still let a NULL-padded row pass.
+ if top_level && *op == Operator::And {
+ left_sides.union(right_sides)
+ } else {
+ // OR (and nested AND): a NULL-padded row is rejected only
+ // if both branches reject NULLs for the same side.
+ left_sides.intersection(right_sides)
}
}
- // Any other operator that DataFusion declares as NULL-on-NULL:
- // recurse into both operands so we collect their columns.
+ // Other NULL-on-NULL operators preserve null rejection from either
+ // operand.
op if op.returns_null_on_null() => {
- extract_null_rejecting_columns(
- left,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- );
- extract_null_rejecting_columns(
- right,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- )
+ let left_sides =
+ extract_null_rejecting_sides(left, left_schema, right_schema, false);
+ let right_sides =
+ extract_null_rejecting_sides(right, left_schema, right_schema, false);
+ left_sides.union(right_sides)
}
- // All other operators (notably including IS [ NOT ] DISTINCT FROM)
- // are declared as not null-propagating, so they don't contribute
- // any null-rejecting columns.
- _ => {}
+ // Other operators, notably IS [ NOT ] DISTINCT FROM, are not
+ // NULL-propagating and provide no side-level rejection evidence.
+ _ => NullRejectingSides::default(),
},
- Expr::Not(arg) | Expr::Negative(arg) => extract_null_rejecting_columns(
- arg,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- ),
- // IS NOT NULL / IS TRUE / IS FALSE / IS NOT UNKNOWN all return FALSE on
- // NULL input. At the top of a WHERE clause, that FALSE filters the row
- // and so we can recurse; below the top level the surrounding context
- // may transform that FALSE into something that accepts NULL rows,
- // making the recursion unsound.
+ Expr::Not(arg) | Expr::Negative(arg) => {
+ extract_null_rejecting_sides(arg, left_schema, right_schema, false)
+ }
+ // These wrappers return FALSE on NULL input, so they reject NULLs only
+ // when they are themselves in the root WHERE context. Under another
+ // expression, that FALSE can be transformed into a NULL-accepting result
+ // (for example by NOT), so recurse only at the top level.
Expr::IsNotNull(arg)
| Expr::IsTrue(arg)
| Expr::IsFalse(arg)
| Expr::IsNotUnknown(arg) => {
- if !top_level {
- return;
+ if top_level {
+ extract_null_rejecting_sides(arg, left_schema, right_schema, false)
+ } else {
+ NullRejectingSides::default()
}
- extract_null_rejecting_columns(
- arg,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- )
}
Expr::Cast(Cast { expr, field: _ })
- | Expr::TryCast(TryCast { expr, field: _ }) => extract_null_rejecting_columns(
- expr,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- ),
- // IN list and BETWEEN are null-rejecting on the input expression:
- // NULL input yields a NULL result, regardless of whether the list
- // or range bounds themselves contain NULLs.
- Expr::InList(InList { expr, .. }) => extract_null_rejecting_columns(
- expr,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- ),
- Expr::Between(between) => extract_null_rejecting_columns(
- &between.expr,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- ),
+ | Expr::TryCast(TryCast { expr, field: _ }) => {
+ extract_null_rejecting_sides(expr, left_schema, right_schema, false)
+ }
+ // IN list and BETWEEN reject NULLs from their input expression; list
+ // values and range bounds do not affect which join side is padded.
+ Expr::InList(InList { expr, .. }) => {
+ extract_null_rejecting_sides(expr, left_schema, right_schema, false)
+ }
+ Expr::Between(between) => {
+ extract_null_rejecting_sides(&between.expr, left_schema, right_schema, false)
+ }
Expr::Like(Like { expr, pattern, .. }) => {
- extract_null_rejecting_columns(
- expr,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- );
- extract_null_rejecting_columns(
- pattern,
- null_rejecting_cols,
- left_schema,
- right_schema,
- false,
- );
+ let expr_sides =
+ extract_null_rejecting_sides(expr, left_schema, right_schema, false);
+ let pattern_sides =
+ extract_null_rejecting_sides(pattern, left_schema, right_schema, false);
+ expr_sides.union(pattern_sides)
}
- // Anything not handled above contributes no null-rejecting
- // columns. Two categories worth calling out:
- // - IS NULL, IS NOT TRUE, IS NOT FALSE, IS UNKNOWN — return
- // TRUE on NULL input, so they actively *accept* NULL rows
- // and are intentionally excluded.
- // - Function calls (scalar / aggregate / window / UDF),
- // scalar subqueries, struct/list accessors, aliases,
- // literals, etc. — we don't have a uniform NULL-propagation
- // guarantee for these cases, so we conservatively skip them.
- _ => {}
+ // Everything else is conservative: NULL-accepting predicates such as
+ // IS NULL / IS NOT TRUE / IS NOT FALSE / IS UNKNOWN must not eliminate
+ // an outer join, and functions/subqueries/accessors/literals have no
+ // uniform NULL-propagation contract here.
+ _ => NullRejectingSides::default(),
}
}
@@ -454,6 +394,46 @@ mod tests {
not, try_cast,
};
+ #[test]
+ fn null_rejecting_sides_union() {
+ let left_side = NullRejectingSides {
+ left: true,
+ right: false,
+ };
+ let right_side = NullRejectingSides {
+ left: false,
+ right: true,
+ };
+
+ assert_eq!(
+ left_side.union(right_side),
+ NullRejectingSides {
+ left: true,
+ right: true,
+ }
+ );
+ }
+
+ #[test]
+ fn null_rejecting_sides_intersection() {
+ let both_sides = NullRejectingSides {
+ left: true,
+ right: true,
+ };
+ let right_side = NullRejectingSides {
+ left: false,
+ right: true,
+ };
+
+ assert_eq!(
+ both_sides.intersection(right_side),
+ NullRejectingSides {
+ left: false,
+ right: true,
+ }
+ );
+ }
+
macro_rules! assert_optimized_plan_equal {
(
$plan:expr,