Apache DataFusion SQL Query Engine
feat: add strictness metadata for scalar UDF null propagation and use it in outer join elimination (#23148)
linfeng66fdcb01cbe8
diff --git a/datafusion/expr/src/predicate_bounds.rs b/datafusion/expr/src/predicate_bounds.rsindex 992d9f88bb..6b672221a7 100644--- a/datafusion/expr/src/predicate_bounds.rs+++ b/datafusion/expr/src/predicate_bounds.rs@@ -183,6 +183,10 @@ impl PredicateBoundsEvaluator<'_> {Expr::BinaryExpr(BinaryExpr { op, .. }) if op.returns_null_on_null() => {self.is_null_if_any_child_null(expr)}+ // Strict scalar functions return NULL when any argument is NULL.+ Expr::ScalarFunction(func) if func.func.is_strict() => {+ self.is_null_if_any_child_null(expr)+ }Expr::Alias(_)| Expr::Cast(_)| Expr::Like(_)@@ -235,8 +239,9 @@ mod tests {use crate::expr::ScalarFunction;use crate::predicate_bounds::evaluate_bounds;use crate::{- Expr, binary_expr, col, create_udf, is_false, is_not_false, is_not_null,- is_not_true, is_not_unknown, is_null, is_true, is_unknown, lit, not,+ Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, binary_expr, col,+ create_udf, is_false, is_not_false, is_not_null, is_not_true, is_not_unknown,+ is_null, is_true, is_unknown, lit, not,};use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::{DFSchema, Result, ScalarValue};@@ -666,6 +671,29 @@ mod tests {}}+ #[test]+ fn evaluate_bounds_strict_udf_is_null_when_child_null() {+ let col = col("col");+ let strict_func = make_test_udf_expr("strict_test", true, vec![col.clone()]);+ let non_strict_func =+ make_test_udf_expr("non_strict_test", false, vec![col.clone()]);+ let schema = DFSchema::try_from(Schema::new(vec![Field::new(+ "col",+ DataType::UInt8,+ true,+ )]))+ .unwrap();++ assert_eq!(+ evaluate_bounds(&is_not_null(strict_func), Some(&col), &schema).unwrap(),+ NullableInterval::FALSE,+ );+ assert_eq!(+ evaluate_bounds(&is_not_null(non_strict_func), Some(&col), &schema).unwrap(),+ NullableInterval::TRUE_OR_FALSE,+ );+ }+fn make_scalar_func_expr() -> Expr {let scalar_func_impl =|_: &[ColumnarValue]| Ok(ColumnarValue::Scalar(ScalarValue::Null));@@ -678,4 +706,51 @@ mod tests {);Expr::ScalarFunction(ScalarFunction::new_udf(Arc::new(udf), vec![]))}++ #[derive(Debug, PartialEq, Eq, Hash)]+ struct TestUdf {+ name: &'static str,+ signature: Signature,+ strict: bool,+ }++ impl TestUdf {+ fn new(name: &'static str, strict: bool) -> Self {+ Self {+ name,+ signature: Signature::uniform(+ 1,+ vec![DataType::UInt8],+ Volatility::Immutable,+ ),+ strict,+ }+ }+ }++ impl ScalarUDFImpl for TestUdf {+ fn name(&self) -> &str {+ self.name+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::UInt8)+ }++ fn is_strict(&self) -> bool {+ self.strict+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ unimplemented!()+ }+ }++ fn make_test_udf_expr(name: &'static str, strict: bool, args: Vec<Expr>) -> Expr {+ ScalarUDF::from(TestUdf::new(name, strict)).call(args)+ }}diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rsindex 6a3aa31a86..e206ce8b29 100644--- a/datafusion/expr/src/udf.rs+++ b/datafusion/expr/src/udf.rs@@ -209,6 +209,14 @@ impl ScalarUDF {self.inner.aliases()}+ /// Returns true if this function always returns NULL when any argument is+ /// NULL.+ ///+ /// See [`ScalarUDFImpl::is_strict`] for more details.+ pub fn is_strict(&self) -> bool {+ self.inner.is_strict()+ }+/// Returns this function's [`Signature`] (what input types are accepted).////// See [`ScalarUDFImpl::signature`] for more details.@@ -693,6 +701,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any {true}+ /// Returns true if this function always returns NULL when any argument is+ /// NULL.+ ///+ /// Strict functions are NULL-propagating: if any argument evaluates to+ /// NULL, the function result is guaranteed to be NULL. Optimizer rules can+ /// use this property when reasoning about expression nullability and+ /// null-rejecting filters.+ ///+ /// Defaults to `false` because user-defined functions may choose to accept+ /// NULL inputs and produce non-NULL results.+ fn is_strict(&self) -> bool {+ false+ }+/// Invoke the function returning the appropriate result.////// # Performance@@ -1103,6 +1125,10 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {self.inner.is_nullable(args, schema)}+ fn is_strict(&self) -> bool {+ self.inner.is_strict()+ }+fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {self.inner.invoke_with_args(args)}diff --git a/datafusion/functions/src/math/abs.rs b/datafusion/functions/src/math/abs.rsindex 02ac89756d..5c5c24a1a6 100644--- a/datafusion/functions/src/math/abs.rs+++ b/datafusion/functions/src/math/abs.rs@@ -158,6 +158,10 @@ impl ScalarUDFImpl for AbsFunc {Ok(arg_types[0].clone())}+ fn is_strict(&self) -> bool {+ true+ }+fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {let args = ColumnarValue::values_to_arrays(&args.args)?;let [input] = take_function_args(self.name(), args)?;diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rsindex 9ec99069f6..a327da66f8 100644--- a/datafusion/optimizer/src/eliminate_outer_join.rs+++ b/datafusion/optimizer/src/eliminate_outer_join.rs@@ -371,10 +371,20 @@ fn extract_null_rejecting_sides(extract_null_rejecting_sides(pattern, left_schema, right_schema, false);expr_sides.union(pattern_sides)}+ // Strict scalar functions are NULL-propagating: if any argument from a+ // padded join side is NULL, the function result is NULL, and an+ // enclosing NULL-rejecting predicate filters the row out.+ Expr::ScalarFunction(func) if func.func.is_strict() => func+ .args+ .iter()+ .map(|arg| {+ extract_null_rejecting_sides(arg, left_schema, right_schema, false)+ })+ .fold(NullRejectingSides::default(), NullRejectingSides::union),// Everything else is conservative: NULL-accepting predicates such as// IS NULL / IS NOT TRUE / IS NOT FALSE / IS UNKNOWN must not eliminate- // an outer join, and functions/subqueries/accessors/literals have no- // uniform NULL-propagation contract here.+ // an outer join, and non-strict functions/subqueries/accessors/literals+ // have no uniform NULL-propagation contract here._ => NullRejectingSides::default(),}}@@ -388,8 +398,10 @@ mod tests {use arrow::datatypes::DataType;use datafusion_common::ScalarValue;use datafusion_expr::{+ ColumnarValue,Operator::{And, Or},- binary_expr, cast, col, lit,+ ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, binary_expr,+ cast, col, lit,logical_plan::builder::LogicalPlanBuilder,not, try_cast,};@@ -450,6 +462,57 @@ mod tests {}};}+ #[derive(Debug, PartialEq, Eq, Hash)]+ struct TestUdf {+ name: &'static str,+ signature: Signature,+ strict: bool,+ }++ impl TestUdf {+ fn new(name: &'static str, strict: bool) -> Self {+ Self {+ name,+ signature: Signature::uniform(+ 1,+ vec![DataType::UInt32],+ Volatility::Immutable,+ ),+ strict,+ }+ }+ }++ impl ScalarUDFImpl for TestUdf {+ fn name(&self) -> &str {+ self.name+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::UInt32)+ }++ fn is_strict(&self) -> bool {+ self.strict+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ unimplemented!()+ }+ }++ fn strict_udf(arg: Expr) -> Expr {+ ScalarUDF::from(TestUdf::new("strict_test", true)).call(vec![arg])+ }++ fn non_strict_udf(arg: Expr) -> Expr {+ ScalarUDF::from(TestUdf::new("non_strict_test", false)).call(vec![arg])+ }+#[test]fn eliminate_left_with_null() -> Result<()> {let t1 = test_table_scan_with_name("t1")?;@@ -498,6 +561,98 @@ mod tests {")}+ #[test]+ fn eliminate_left_with_strict_function() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(strict_udf(col("t2.b")).gt(lit(5u32)))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: strict_test(t2.b) > UInt32(5)+ Inner Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_non_strict_function() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(non_strict_udf(col("t2.b")).gt(lit(5u32)))?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: non_strict_test(t2.b) > UInt32(5)+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn eliminate_left_with_nested_strict_is_not_null() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(strict_udf(strict_udf(col("t2.b"))).is_not_null())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: strict_test(strict_test(t2.b)) IS NOT NULL+ Inner Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn no_eliminate_left_with_strict_function_is_null() -> Result<()> {+ let t1 = test_table_scan_with_name("t1")?;+ let t2 = test_table_scan_with_name("t2")?;++ let plan = LogicalPlanBuilder::from(t1)+ .join(+ t2,+ JoinType::Left,+ (vec![Column::from_name("a")], vec![Column::from_name("a")]),+ None,+ )?+ .filter(strict_udf(col("t2.b")).is_null())?+ .build()?;++ assert_optimized_plan_equal!(plan, @r"+ Filter: strict_test(t2.b) IS NULL+ Left Join: t1.a = t2.a+ TableScan: t1+ TableScan: t2+ ")+ }+#[test]fn eliminate_right_with_or() -> Result<()> {let t1 = test_table_scan_with_name("t1")?;diff --git a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt b/datafusion/sqllogictest/test_files/eliminate_outer_join.sltindex 584d8af419..52ae3e37ef 100644--- a/datafusion/sqllogictest/test_files/eliminate_outer_join.slt+++ b/datafusion/sqllogictest/test_files/eliminate_outer_join.slt@@ -642,6 +642,81 @@ select s.a from s where s.y > 150;----2+# https://github.com/apache/datafusion/issues/13232+# Strict scalar functions over the nullable side of an outer join reject the+# NULL-padding rows when used in a null-rejecting filter.+query TT+explain+select t1.a+from t1 left join t2 on t1.a = t2.x+where abs(t2.y) > 5;+----+logical_plan+01)Projection: t1.a+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----Projection: t2.x+05)------Filter: abs(t2.y) > Int32(5)+06)--------TableScan: t2 projection=[x, y]++query I rowsort+select t1.a+from t1 left join t2 on t1.a = t2.x+where abs(t2.y) > 5;+----+1+2++query TT+explain+select t1.a+from t1 inner join t2 on t1.a = t2.x+where abs(t2.y) > 5;+----+logical_plan+01)Projection: t1.a+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----Projection: t2.x+05)------Filter: abs(t2.y) > Int32(5)+06)--------TableScan: t2 projection=[x, y]++query I rowsort+select t1.a+from t1 inner join t2 on t1.a = t2.x+where abs(t2.y) > 5;+----+1+2++query TT+explain+select t1.a+from t1 left join t2 on t1.a = t2.x+where abs(t2.y) is not null;+----+logical_plan+01)Projection: t1.a+02)--Inner Join: t1.a = t2.x+03)----TableScan: t1 projection=[a]+04)----Projection: t2.x+05)------Filter: abs(t2.y) IS NOT NULL+06)--------TableScan: t2 projection=[x, y]++query TT+explain+select t1.a+from t1 left join t2 on t1.a = t2.x+where coalesce(t2.y, 0) > 5;+----+logical_plan+01)Projection: t1.a+02)--Filter: CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Int64(0) END > Int64(5)+03)----Projection: CAST(t2.y AS Int64) AS __common_expr_3, t1.a+04)------Left Join: t1.a = t2.x+05)--------TableScan: t1 projection=[a]+06)--------TableScan: t2 projection=[x, y]+###### Cleanup###