Apache DataFusion SQL Query Engine
fix: Add custom nullability for Spark ILIKE function (#19206)
Eshaan Guptaaebae990dcfb
diff --git a/datafusion/spark/src/function/string/ilike.rs b/datafusion/spark/src/function/string/ilike.rsindex a160749523..4be2969248 100644--- a/datafusion/spark/src/function/string/ilike.rs+++ b/datafusion/spark/src/function/string/ilike.rs@@ -17,10 +17,12 @@use arrow::array::ArrayRef;use arrow::compute::ilike;-use arrow::datatypes::DataType;-use datafusion_common::{exec_err, Result};+use arrow::datatypes::{DataType, Field};+use datafusion_common::{exec_err, internal_err, Result};use datafusion_expr::ColumnarValue;-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};+use datafusion_expr::{+ ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+};use datafusion_functions::utils::make_scalar_function;use std::any::Any;use std::sync::Arc;@@ -60,7 +62,14 @@ impl ScalarUDFImpl for SparkILike {}fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(DataType::Boolean)+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<Arc<Field>> {+ // ILIKE returns a boolean value+ // The result is nullable if any of the input arguments is nullable+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(Field::new("ilike", DataType::Boolean, nullable)))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -83,9 +92,9 @@ mod tests {use super::*;use crate::function::utils::test::test_scalar_function;use arrow::array::{Array, BooleanArray};- use arrow::datatypes::DataType::Boolean;+ use arrow::datatypes::{DataType::Boolean, Field};use datafusion_common::{Result, ScalarValue};- use datafusion_expr::{ColumnarValue, ScalarUDFImpl};+ use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarUDFImpl};macro_rules! test_ilike_string_invoke {($INPUT1:expr, $INPUT2:expr, $EXPECTED:expr) => {@@ -170,4 +179,73 @@ mod tests {Ok(())}++ #[test]+ fn test_ilike_nullability() {+ let ilike = SparkILike::new();++ // Test with non-nullable arguments+ let non_nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, false));+ let non_nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, false));++ let result = ilike+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&non_nullable_field1),+ Arc::clone(&non_nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should not be nullable when both inputs are non-nullable+ assert!(!result.is_nullable());+ assert_eq!(result.data_type(), &Boolean);++ // Test with first argument nullable+ let nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, true));++ let result = ilike+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&nullable_field1),+ Arc::clone(&non_nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when first input is nullable+ assert!(result.is_nullable());+ assert_eq!(result.data_type(), &Boolean);++ // Test with second argument nullable+ let nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, true));++ let result = ilike+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&non_nullable_field1),+ Arc::clone(&nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when second input is nullable+ assert!(result.is_nullable());+ assert_eq!(result.data_type(), &Boolean);++ // Test with both arguments nullable+ let result = ilike+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&nullable_field1), Arc::clone(&nullable_field2)],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when both inputs are nullable+ assert!(result.is_nullable());+ assert_eq!(result.data_type(), &Boolean);+ }}