Skip to content

Apache DataFusion SQL Query Engine

fix: Add custom nullability for Spark ILIKE function (#19206)

Eshaan Guptaaebae990dcfb
diff --git a/datafusion/spark/src/function/string/ilike.rs b/datafusion/spark/src/function/string/ilike.rs
index a160749523..4be2969248 100644
--- a/datafusion/spark/src/function/string/ilike.rs
+++ b/datafusion/spark/src/function/string/ilike.rs
@@ -17,10 +17,12 @@
use arrow::array::ArrayRef;
use arrow::compute::ilike;
-use arrow::datatypes::DataType;
-use datafusion_common::{exec_err, Result};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::{exec_err, internal_err, Result};
use datafusion_expr::ColumnarValue;
-use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
+use datafusion_expr::{
+ ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
use datafusion_functions::utils::make_scalar_function;
use std::any::Any;
use std::sync::Arc;
@@ -60,7 +62,14 @@ impl ScalarUDFImpl for SparkILike {
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Boolean)
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<Arc<Field>> {
+ // ILIKE returns a boolean value
+ // The result is nullable if any of the input arguments is nullable
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ Ok(Arc::new(Field::new("ilike", DataType::Boolean, nullable)))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -83,9 +92,9 @@ mod tests {
use super::*;
use crate::function::utils::test::test_scalar_function;
use arrow::array::{Array, BooleanArray};
- use arrow::datatypes::DataType::Boolean;
+ use arrow::datatypes::{DataType::Boolean, Field};
use datafusion_common::{Result, ScalarValue};
- use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
+ use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarUDFImpl};
macro_rules! test_ilike_string_invoke {
($INPUT1:expr, $INPUT2:expr, $EXPECTED:expr) => {
@@ -170,4 +179,73 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_ilike_nullability() {
+ let ilike = SparkILike::new();
+
+ // Test with non-nullable arguments
+ let non_nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, false));
+ let non_nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, false));
+
+ let result = ilike
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&non_nullable_field1),
+ Arc::clone(&non_nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should not be nullable when both inputs are non-nullable
+ assert!(!result.is_nullable());
+ assert_eq!(result.data_type(), &Boolean);
+
+ // Test with first argument nullable
+ let nullable_field1 = Arc::new(Field::new("str", DataType::Utf8, true));
+
+ let result = ilike
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&nullable_field1),
+ Arc::clone(&non_nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when first input is nullable
+ assert!(result.is_nullable());
+ assert_eq!(result.data_type(), &Boolean);
+
+ // Test with second argument nullable
+ let nullable_field2 = Arc::new(Field::new("pattern", DataType::Utf8, true));
+
+ let result = ilike
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[
+ Arc::clone(&non_nullable_field1),
+ Arc::clone(&nullable_field2),
+ ],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when second input is nullable
+ assert!(result.is_nullable());
+ assert_eq!(result.data_type(), &Boolean);
+
+ // Test with both arguments nullable
+ let result = ilike
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[Arc::clone(&nullable_field1), Arc::clone(&nullable_field2)],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ // The result should be nullable when both inputs are nullable
+ assert!(result.is_nullable());
+ assert_eq!(result.data_type(), &Boolean);
+ }
}