Skip to content

Apache DataFusion SQL Query Engine

fix: Add custom nullability for Spark LIKE function (#19218)

Karan Pradhan902d3b32be4d
diff --git a/datafusion/spark/src/function/string/like.rs b/datafusion/spark/src/function/string/like.rs
index 4e9b523279..ffa1737023 100644
--- a/datafusion/spark/src/function/string/like.rs
+++ b/datafusion/spark/src/function/string/like.rs
@@ -17,10 +17,12 @@
use arrow::array::ArrayRef;
use arrow::compute::like;
-use arrow::datatypes::DataType;
-use datafusion_common::{Result, exec_err};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, exec_err, internal_err};
use datafusion_expr::ColumnarValue;
-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
+use datafusion_expr::{
+ ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
use datafusion_functions::utils::make_scalar_function;
use std::any::Any;
use std::sync::Arc;
@@ -60,7 +62,16 @@ impl ScalarUDFImpl for SparkLike {
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Boolean)
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ Ok(Arc::new(Field::new(
+ self.name(),
+ DataType::Boolean,
+ nullable,
+ )))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -83,9 +94,9 @@ mod tests {
use super::*;
use crate::function::utils::test::test_scalar_function;
use arrow::array::{Array, BooleanArray};
- use arrow::datatypes::DataType::Boolean;
+ use arrow::datatypes::{DataType::Boolean, Field};
use datafusion_common::{Result, ScalarValue};
- use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
+ use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarUDFImpl};
macro_rules! test_like_string_invoke {
($INPUT1:expr, $INPUT2:expr, $EXPECTED:expr) => {
@@ -175,4 +186,73 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_like_nullability() {
+ let like = SparkLike::new();
+
+ // Test with non-nullable arguments
+ let non_nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, false));
+ let non_nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, false));
+
+ let both_non_nullable = like
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&non_nullable_field1),
+ Arc::clone(&non_nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should not be nullable when both inputs are non-nullable
+ assert!(!both_non_nullable.is_nullable());
+ assert_eq!(both_non_nullable.data_type(), &Boolean);
+
+ // Test with first argument nullable
+ let nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, true));
+
+ let first_nullable = like
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&nullable_field1),
+ Arc::clone(&non_nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when first input is nullable
+ assert!(first_nullable.is_nullable());
+ assert_eq!(first_nullable.data_type(), &Boolean);
+
+ // Test with second argument nullable
+ let nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, true));
+
+ let second_nullable = like
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&non_nullable_field1),
+ Arc::clone(&nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when second input is nullable
+ assert!(second_nullable.is_nullable());
+ assert_eq!(second_nullable.data_type(), &Boolean);
+
+ // Test with both arguments nullable
+ let first_second_nullable = like
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[Arc::clone(&nullable_field1), Arc::clone(&nullable_field2)],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when both inputs are nullable
+ assert!(first_second_nullable.is_nullable());
+ assert_eq!(first_second_nullable.data_type(), &Boolean);
+ }
}