Skip to content

Apache DataFusion SQL Query Engine

fix: derive custom nullable for spark `date_sub` (#19225)

Kumar Ujjawal49cfee05c249
diff --git a/datafusion/spark/src/function/datetime/date_sub.rs b/datafusion/spark/src/function/datetime/date_sub.rs
index a3b26661d1..df406ba7ef 100644
--- a/datafusion/spark/src/function/datetime/date_sub.rs
+++ b/datafusion/spark/src/function/datetime/date_sub.rs
@@ -20,15 +20,15 @@ use std::sync::Arc;
use arrow::array::ArrayRef;
use arrow::compute;
-use arrow::datatypes::{DataType, Date32Type};
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef};
use arrow::error::ArrowError;
use datafusion_common::cast::{
as_date32_array, as_int16_array, as_int32_array, as_int8_array,
};
use datafusion_common::{internal_err, Result};
use datafusion_expr::{
- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
- Volatility,
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ TypeSignature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
@@ -72,7 +72,21 @@ impl ScalarUDFImpl for SparkDateSub {
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Date32)
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable())
+ || args
+ .scalar_arguments
+ .iter()
+ .any(|arg| matches!(arg, Some(sv) if sv.is_null()));
+
+ Ok(Arc::new(Field::new(
+ self.name(),
+ DataType::Date32,
+ nullable,
+ )))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -134,3 +148,61 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result<ArrayRef> {
};
Ok(Arc::new(result))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use datafusion_common::ScalarValue;
+
+ #[test]
+ fn test_date_sub_nullability_non_nullable_args() {
+ let udf = SparkDateSub::new();
+ let date_field = Arc::new(Field::new("d", DataType::Date32, false));
+ let days_field = Arc::new(Field::new("n", DataType::Int32, false));
+
+ let result = udf
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[date_field, days_field],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ assert!(!result.is_nullable());
+ assert_eq!(result.data_type(), &DataType::Date32);
+ }
+
+ #[test]
+ fn test_date_sub_nullability_nullable_arg() {
+ let udf = SparkDateSub::new();
+ let date_field = Arc::new(Field::new("d", DataType::Date32, false));
+ let nullable_days_field = Arc::new(Field::new("n", DataType::Int32, true));
+
+ let result = udf
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[date_field, nullable_days_field],
+ scalar_arguments: &[None, None],
+ })
+ .unwrap();
+
+ assert!(result.is_nullable());
+ assert_eq!(result.data_type(), &DataType::Date32);
+ }
+
+ #[test]
+ fn test_date_sub_nullability_scalar_null_argument() {
+ let udf = SparkDateSub::new();
+ let date_field = Arc::new(Field::new("d", DataType::Date32, false));
+ let days_field = Arc::new(Field::new("n", DataType::Int32, false));
+ let null_scalar = ScalarValue::Int32(None);
+
+ let result = udf
+ .return_field_from_args(ReturnFieldArgs {
+ arg_fields: &[date_field, days_field],
+ scalar_arguments: &[None, Some(&null_scalar)],
+ })
+ .unwrap();
+
+ assert!(result.is_nullable());
+ assert_eq!(result.data_type(), &DataType::Date32);
+ }
+}