Apache DataFusion SQL Query Engine
fix: Add custom nullability for Spark LIKE function (#19218)
Karan Pradhan902d3b32be4d
diff --git a/datafusion/spark/src/function/string/like.rs b/datafusion/spark/src/function/string/like.rsindex 4e9b523279..ffa1737023 100644--- a/datafusion/spark/src/function/string/like.rs+++ b/datafusion/spark/src/function/string/like.rs@@ -17,10 +17,12 @@use arrow::array::ArrayRef;use arrow::compute::like;-use arrow::datatypes::DataType;-use datafusion_common::{Result, exec_err};+use arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::{Result, exec_err, internal_err};use datafusion_expr::ColumnarValue;-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};+use datafusion_expr::{+ ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+};use datafusion_functions::utils::make_scalar_function;use std::any::Any;use std::sync::Arc;@@ -60,7 +62,16 @@ impl ScalarUDFImpl for SparkLike {}fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(DataType::Boolean)+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(Field::new(+ self.name(),+ DataType::Boolean,+ nullable,+ )))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -83,9 +94,9 @@ mod tests {use super::*;use crate::function::utils::test::test_scalar_function;use arrow::array::{Array, BooleanArray};- use arrow::datatypes::DataType::Boolean;+ use arrow::datatypes::{DataType::Boolean, Field};use datafusion_common::{Result, ScalarValue};- use datafusion_expr::{ColumnarValue, ScalarUDFImpl};+ use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarUDFImpl};macro_rules! test_like_string_invoke {($INPUT1:expr, $INPUT2:expr, $EXPECTED:expr) => {@@ -175,4 +186,73 @@ mod tests {Ok(())}++ #[test]+ fn test_like_nullability() {+ let like = SparkLike::new();++ // Test with non-nullable arguments+ let non_nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, false));+ let non_nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, false));++ let both_non_nullable = like+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&non_nullable_field1),+ Arc::clone(&non_nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should not be nullable when both inputs are non-nullable+ assert!(!both_non_nullable.is_nullable());+ assert_eq!(both_non_nullable.data_type(), &Boolean);++ // Test with first argument nullable+ let nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, true));++ let first_nullable = like+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&nullable_field1),+ Arc::clone(&non_nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when first input is nullable+ assert!(first_nullable.is_nullable());+ assert_eq!(first_nullable.data_type(), &Boolean);++ // Test with second argument nullable+ let nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, true));++ let second_nullable = like+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[+ Arc::clone(&non_nullable_field1),+ Arc::clone(&nullable_field2),+ ],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when second input is nullable+ assert!(second_nullable.is_nullable());+ assert_eq!(second_nullable.data_type(), &Boolean);++ // Test with both arguments nullable+ let first_second_nullable = like+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &[Arc::clone(&nullable_field1), Arc::clone(&nullable_field2)],+ scalar_arguments: &[None, None],+ })+ .unwrap();++ // The result should be nullable when both inputs are nullable+ assert!(first_second_nullable.is_nullable());+ assert_eq!(first_second_nullable.data_type(), &Boolean);+ }}