Skip to content

Apache DataFusion SQL Query Engine

feat: add strictness metadata for scalar UDF null propagation and use it in outer join elimination (#23148)

linfeng66fdcb01cbe8
diff --git a/datafusion/expr/src/predicate_bounds.rs b/datafusion/expr/src/predicate_bounds.rs
index 992d9f88bb..6b672221a7 100644
--- a/datafusion/expr/src/predicate_bounds.rs
+++ b/datafusion/expr/src/predicate_bounds.rs
@@ -183,6 +183,10 @@ impl PredicateBoundsEvaluator<'_> {
Expr::BinaryExpr(BinaryExpr { op, .. }) if op.returns_null_on_null() => {
self.is_null_if_any_child_null(expr)
}
+ // Strict scalar functions return NULL when any argument is NULL.
+ Expr::ScalarFunction(func) if func.func.is_strict() => {
+ self.is_null_if_any_child_null(expr)
+ }
Expr::Alias(_)
| Expr::Cast(_)
| Expr::Like(_)
@@ -235,8 +239,9 @@ mod tests {
use crate::expr::ScalarFunction;
use crate::predicate_bounds::evaluate_bounds;
use crate::{
- Expr, binary_expr, col, create_udf, is_false, is_not_false, is_not_null,
- is_not_true, is_not_unknown, is_null, is_true, is_unknown, lit, not,
+ Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, binary_expr, col,
+ create_udf, is_false, is_not_false, is_not_null, is_not_true, is_not_unknown,
+ is_null, is_true, is_unknown, lit, not,
};
use arrow::datatypes::{DataType, Field, Schema};
use datafusion_common::{DFSchema, Result, ScalarValue};
@@ -666,6 +671,29 @@ mod tests {
}
}
+ #[test]
+ fn evaluate_bounds_strict_udf_is_null_when_child_null() {
+ let col = col("col");
+ let strict_func = make_test_udf_expr("strict_test", true, vec![col.clone()]);
+ let non_strict_func =
+ make_test_udf_expr("non_strict_test", false, vec![col.clone()]);
+ let schema = DFSchema::try_from(Schema::new(vec![Field::new(
+ "col",
+ DataType::UInt8,
+ true,
+ )]))
+ .unwrap();
+
+ assert_eq!(
+ evaluate_bounds(&is_not_null(strict_func), Some(&col), &schema).unwrap(),
+ NullableInterval::FALSE,
+ );
+ assert_eq!(
+ evaluate_bounds(&is_not_null(non_strict_func), Some(&col), &schema).unwrap(),
+ NullableInterval::TRUE_OR_FALSE,
+ );
+ }
+
fn make_scalar_func_expr() -> Expr {
let scalar_func_impl =
|_: &[ColumnarValue]| Ok(ColumnarValue::Scalar(ScalarValue::Null));
@@ -678,4 +706,51 @@ mod tests {
);
Expr::ScalarFunction(ScalarFunction::new_udf(Arc::new(udf), vec![]))
}
+
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct TestUdf {
+ name: &'static str,
+ signature: Signature,
+ strict: bool,
+ }
+
+ impl TestUdf {
+ fn new(name: &'static str, strict: bool) -> Self {
+ Self {
+ name,
+ signature: Signature::uniform(
+ 1,
+ vec![DataType::UInt8],
+ Volatility::Immutable,
+ ),
+ strict,
+ }
+ }
+ }
+
+ impl ScalarUDFImpl for TestUdf {
+ fn name(&self) -> &str {
+ self.name
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::UInt8)
+ }
+
+ fn is_strict(&self) -> bool {
+ self.strict
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ unimplemented!()
+ }
+ }
+
+ fn make_test_udf_expr(name: &'static str, strict: bool, args: Vec<Expr>) -> Expr {
+ ScalarUDF::from(TestUdf::new(name, strict)).call(args)
+ }
}
diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs
index 6a3aa31a86..e206ce8b29 100644
--- a/datafusion/expr/src/udf.rs
+++ b/datafusion/expr/src/udf.rs
@@ -209,6 +209,14 @@ impl ScalarUDF {
self.inner.aliases()
}
+ /// Returns true if this function always returns NULL when any argument is
+ /// NULL.
+ ///
+ /// See [`ScalarUDFImpl::is_strict`] for more details.
+ pub fn is_strict(&self) -> bool {
+ self.inner.is_strict()
+ }
+
/// Returns this function's [`Signature`] (what input types are accepted).
///
/// See [`ScalarUDFImpl::signature`] for more details.
@@ -693,6 +701,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {
true
}
+ /// Returns true if this function always returns NULL when any argument is
+ /// NULL.
+ ///
+ /// Strict functions are NULL-propagating: if any argument evaluates to
+ /// NULL, the function result is guaranteed to be NULL. Optimizer rules can
+ /// use this property when reasoning about expression nullability and
+ /// null-rejecting filters.
+ ///
+ /// Defaults to `false` because user-defined functions may choose to accept
+ /// NULL inputs and produce non-NULL results.
+ fn is_strict(&self) -> bool {
+ false
+ }
+
/// Invoke the function returning the appropriate result.
///
/// # Performance
@@ -1103,6 +1125,10 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {
self.inner.is_nullable(args, schema)
}
+ fn is_strict(&self) -> bool {
+ self.inner.is_strict()
+ }
+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
self.inner.invoke_with_args(args)
}
diff --git a/datafusion/functions/src/math/abs.rs b/datafusion/functions/src/math/abs.rs
index 02ac89756d..5c5c24a1a6 100644
--- a/datafusion/functions/src/math/abs.rs
+++ b/datafusion/functions/src/math/abs.rs
@@ -158,6 +158,10 @@ impl ScalarUDFImpl for AbsFunc {
Ok(arg_types[0].clone())
}
+ fn is_strict(&self) -> bool {
+ true
+ }
+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let args = ColumnarValue::values_to_arrays(&args.args)?;
let [input] = take_function_args(self.name(), args)?;
diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rs
index 9ec99069f6..a327da66f8 100644
--- a/datafusion/optimizer/src/eliminate_outer_join.rs
+++ b/datafusion/optimizer/src/eliminate_outer_join.rs
@@ -371,10 +371,20 @@ fn extract_null_rejecting_sides(
extract_null_rejecting_sides(pattern, left_schema, right_schema, false);
expr_sides.union(pattern_sides)
}
+ // Strict scalar functions are NULL-propagating: if any argument from a
+ // padded join side is NULL, the function result is NULL, and an
+ // enclosing NULL-rejecting predicate filters the row out.
+ Expr::ScalarFunction(func) if func.func.is_strict() => func
+ .args
+ .iter()
+ .map(|arg| {
+ extract_null_rejecting_sides(arg, left_schema, right_schema, false)
+ })
+ .fold(NullRejectingSides::default(), NullRejectingSides::union),
// Everything else is conservative: NULL-accepting predicates such as
// IS NULL / IS NOT TRUE / IS NOT FALSE / IS UNKNOWN must not eliminate
- // an outer join, and functions/subqueries/accessors/literals have no
- // uniform NULL-propagation contract here.
+ // an outer join, and non-strict functions/subqueries/accessors/literals
+ // have no uniform NULL-propagation contract here.
_ => NullRejectingSides::default(),
}
}
@@ -388,8 +398,10 @@ mod tests {
use arrow::datatypes::DataType;
use datafusion_common::ScalarValue;
use datafusion_expr::{
+ ColumnarValue,
Operator::{And, Or},
- binary_expr, cast, col, lit,
+ ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, binary_expr,
+ cast, col, lit,
logical_plan::builder::LogicalPlanBuilder,
not, try_cast,
};
@@ -450,6 +462,57 @@ mod tests {
}};
}
+ #[derive(Debug, PartialEq, Eq, Hash)]
+ struct TestUdf {
+ name: &'static str,
+ signature: Signature,
+ strict: bool,
+ }
+
+ impl TestUdf {
+ fn new(name: &'static str, strict: bool) -> Self {
+ Self {
+ name,
+ signature: Signature::uniform(
+ 1,
+ vec![DataType::UInt32],
+ Volatility::Immutable,
+ ),
+ strict,
+ }
+ }
+ }
+
+ impl ScalarUDFImpl for TestUdf {
+ fn name(&self) -> &str {
+ self.name
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::UInt32)
+ }
+
+ fn is_strict(&self) -> bool {
+ self.strict
+ }
+
+ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ unimplemented!()
+ }
+ }
+
+ fn strict_udf(arg: Expr) -> Expr {
+ ScalarUDF::from(TestUdf::new("strict_test", true)).call(vec![arg])
+ }
+
+ fn non_strict_udf(arg: Expr) -> Expr {
+ ScalarUDF::from(TestUdf::new("non_strict_test", false)).call(vec![arg])
+ }
+
#[test]
fn eliminate_left_with_null() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
@@ -498,6 +561,98 @@ mod tests {
")
}
+ #[test]
+ fn eliminate_left_with_strict_function() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ let plan = LogicalPlanBuilder::from(t1)
+ .join(
+ t2,
+ JoinType::Left,
+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),
+ None,
+ )?
+ .filter(strict_udf(col("t2.b")).gt(lit(5u32)))?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: strict_test(t2.b) > UInt32(5)
+ Inner Join: t1.a = t2.a
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ #[test]
+ fn no_eliminate_left_with_non_strict_function() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ let plan = LogicalPlanBuilder::from(t1)
+ .join(
+ t2,
+ JoinType::Left,
+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),
+ None,
+ )?
+ .filter(non_strict_udf(col("t2.b")).gt(lit(5u32)))?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: non_strict_test(t2.b) > UInt32(5)
+ Left Join: t1.a = t2.a
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ #[test]
+ fn eliminate_left_with_nested_strict_is_not_null() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ let plan = LogicalPlanBuilder::from(t1)
+ .join(
+ t2,
+ JoinType::Left,
+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),
+ None,
+ )?
+ .filter(strict_udf(strict_udf(col("t2.b"))).is_not_null())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: strict_test(strict_test(t2.b)) IS NOT NULL
+ Inner Join: t1.a = t2.a
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ #[test]
+ fn no_eliminate_left_with_strict_function_is_null() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ let plan = LogicalPlanBuilder::from(t1)
+ .join(
+ t2,
+ JoinType::Left,
+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),
+ None,
+ )?
+ .filter(strict_udf(col("t2.b")).is_null())?
+ .build()?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ Filter: strict_test(t2.b) IS NULL
+ Left Join: t1.a = t2.a
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
#[test]
fn eliminate_right_with_or() -> Result<()> {
let t1 = test_table_scan_with_name("t1")?;
diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt
index 584d8af419..52ae3e37ef 100644
--- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt
+++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt
@@ -642,6 +642,81 @@ select s.a from s where s.y > 150;
----
2
+# https://github.com/apache/datafusion/issues/13232
+# Strict scalar functions over the nullable side of an outer join reject the
+# NULL-padding rows when used in a null-rejecting filter.
+query TT
+explain
+select t1.a
+from t1 left join t2 on t1.a = t2.x
+where abs(t2.y) > 5;
+----
+logical_plan
+01)Projection: t1.a
+02)--Inner Join: t1.a = t2.x
+03)----TableScan: t1 projection=[a]
+04)----Projection: t2.x
+05)------Filter: abs(t2.y) > Int32(5)
+06)--------TableScan: t2 projection=[x, y]
+
+query I rowsort
+select t1.a
+from t1 left join t2 on t1.a = t2.x
+where abs(t2.y) > 5;
+----
+1
+2
+
+query TT
+explain
+select t1.a
+from t1 inner join t2 on t1.a = t2.x
+where abs(t2.y) > 5;
+----
+logical_plan
+01)Projection: t1.a
+02)--Inner Join: t1.a = t2.x
+03)----TableScan: t1 projection=[a]
+04)----Projection: t2.x
+05)------Filter: abs(t2.y) > Int32(5)
+06)--------TableScan: t2 projection=[x, y]
+
+query I rowsort
+select t1.a
+from t1 inner join t2 on t1.a = t2.x
+where abs(t2.y) > 5;
+----
+1
+2
+
+query TT
+explain
+select t1.a
+from t1 left join t2 on t1.a = t2.x
+where abs(t2.y) is not null;
+----
+logical_plan
+01)Projection: t1.a
+02)--Inner Join: t1.a = t2.x
+03)----TableScan: t1 projection=[a]
+04)----Projection: t2.x
+05)------Filter: abs(t2.y) IS NOT NULL
+06)--------TableScan: t2 projection=[x, y]
+
+query TT
+explain
+select t1.a
+from t1 left join t2 on t1.a = t2.x
+where coalesce(t2.y, 0) > 5;
+----
+logical_plan
+01)Projection: t1.a
+02)--Filter: CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Int64(0) END > Int64(5)
+03)----Projection: CAST(t2.y AS Int64) AS __common_expr_3, t1.a
+04)------Left Join: t1.a = t2.x
+05)--------TableScan: t1 projection=[a]
+06)--------TableScan: t2 projection=[x, y]
+
###
### Cleanup
###