Apache DataFusion SQL Query Engine
Refactor `nvl2` Function to Support Lazy Evaluation and Simplification via CASE Expression (#18191)
kosiew22c4214fe1ca
diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rsindex b664fccdfa..d95eb38c19 100644--- a/datafusion/core/tests/dataframe/dataframe_functions.rs+++ b/datafusion/core/tests/dataframe/dataframe_functions.rs@@ -274,6 +274,33 @@ async fn test_nvl2() -> Result<()> {Ok(())}++#[tokio::test]+async fn test_nvl2_short_circuit() -> Result<()> {+ let expr = nvl2(+ col("a"),+ arrow_cast(lit("1"), lit("Int32")),+ arrow_cast(col("a"), lit("Int32")),+ );++ let batches = get_batches(expr).await?;++ assert_snapshot!(+ batches_to_string(&batches),+ @r#"+ +-----------------------------------------------------------------------------------++ | nvl2(test.a,arrow_cast(Utf8("1"),Utf8("Int32")),arrow_cast(test.a,Utf8("Int32"))) |+ +-----------------------------------------------------------------------------------++ | 1 |+ | 1 |+ | 1 |+ | 1 |+ +-----------------------------------------------------------------------------------++ "#+ );++ Ok(())+}#[tokio::test]async fn test_fn_arrow_typeof() -> Result<()> {let expr = arrow_typeof(col("l"));diff --git a/datafusion/core/tests/expr_api/mod.rs b/datafusion/core/tests/expr_api/mod.rsindex 4aee274de9..84e644480a 100644--- a/datafusion/core/tests/expr_api/mod.rs+++ b/datafusion/core/tests/expr_api/mod.rs@@ -320,6 +320,26 @@ async fn test_create_physical_expr() {create_simplified_expr_test(lit(1i32) + lit(2i32), "3");}+#[test]+fn test_create_physical_expr_nvl2() {+ let batch = &TEST_BATCH;+ let df_schema = DFSchema::try_from(batch.schema()).unwrap();+ let ctx = SessionContext::new();++ let expect_err = |expr| {+ let physical_expr = ctx.create_physical_expr(expr, &df_schema).unwrap();+ let err = physical_expr.evaluate(batch).unwrap_err();+ assert!(+ err.to_string()+ .contains("nvl2 should have been simplified to case"),+ "unexpected error: {err:?}"+ );+ };++ expect_err(nvl2(col("i"), lit(1i64), lit(0i64)));+ expect_err(nvl2(lit(1i64), col("i"), lit(0i64)));+}+#[tokio::test]async fn test_create_physical_expr_coercion() {// create_physical_expr does apply type coercion and unwrapping in castdiff --git a/datafusion/functions/src/core/nvl2.rs b/datafusion/functions/src/core/nvl2.rsindex 82aa8d2a4c..45cb6760d0 100644--- a/datafusion/functions/src/core/nvl2.rs+++ b/datafusion/functions/src/core/nvl2.rs@@ -15,17 +15,16 @@// specific language governing permissions and limitations// under the License.-use arrow::array::Array;-use arrow::compute::is_not_null;-use arrow::compute::kernels::zip::zip;-use arrow::datatypes::DataType;+use arrow::datatypes::{DataType, Field, FieldRef};use datafusion_common::{internal_err, utils::take_function_args, Result};use datafusion_expr::{- type_coercion::binary::comparison_coercion, ColumnarValue, Documentation,- ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ conditional_expressions::CaseBuilder,+ simplify::{ExprSimplifyResult, SimplifyInfo},+ type_coercion::binary::comparison_coercion,+ ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,+ ScalarUDFImpl, Signature, Volatility,};use datafusion_macros::user_doc;-use std::sync::Arc;#[user_doc(doc_section(label = "Conditional Functions"),@@ -95,8 +94,37 @@ impl ScalarUDFImpl for NVL2Func {Ok(arg_types[1].clone())}- fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- nvl2_func(&args.args)+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let nullable =+ args.arg_fields[1].is_nullable() || args.arg_fields[2].is_nullable();+ let return_type = args.arg_fields[1].data_type().clone();+ Ok(Field::new(self.name(), return_type, nullable).into())+ }++ fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ internal_err!("nvl2 should have been simplified to case")+ }++ fn simplify(+ &self,+ args: Vec<Expr>,+ _info: &dyn SimplifyInfo,+ ) -> Result<ExprSimplifyResult> {+ let [test, if_non_null, if_null] = take_function_args(self.name(), args)?;++ let expr = CaseBuilder::new(+ None,+ vec![test.is_not_null()],+ vec![if_non_null],+ Some(Box::new(if_null)),+ )+ .end()?;++ Ok(ExprSimplifyResult::Simplified(expr))+ }++ fn short_circuits(&self) -> bool {+ true}fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {@@ -123,42 +151,3 @@ impl ScalarUDFImpl for NVL2Func {self.doc()}}--fn nvl2_func(args: &[ColumnarValue]) -> Result<ColumnarValue> {- let mut len = 1;- let mut is_array = false;- for arg in args {- if let ColumnarValue::Array(array) = arg {- len = array.len();- is_array = true;- break;- }- }- if is_array {- let args = args- .iter()- .map(|arg| match arg {- ColumnarValue::Scalar(scalar) => scalar.to_array_of_size(len),- ColumnarValue::Array(array) => Ok(Arc::clone(array)),- })- .collect::<Result<Vec<_>>>()?;- let [tested, if_non_null, if_null] = take_function_args("nvl2", args)?;- let to_apply = is_not_null(&tested)?;- let value = zip(&to_apply, &if_non_null, &if_null)?;- Ok(ColumnarValue::Array(value))- } else {- let [tested, if_non_null, if_null] = take_function_args("nvl2", args)?;- match &tested {- ColumnarValue::Array(_) => {- internal_err!("except Scalar value, but got Array")- }- ColumnarValue::Scalar(scalar) => {- if scalar.is_null() {- Ok(if_null.clone())- } else {- Ok(if_non_null.clone())- }- }- }- }-}