Apache DataFusion SQL Query Engine
fix: derive custom nullable for spark `make_dt_interval` (#19236)
Kumar Ujjawal2c6f6d369624
diff --git a/datafusion/spark/src/function/datetime/make_dt_interval.rs b/datafusion/spark/src/function/datetime/make_dt_interval.rsindex cb8f6e1d79..f3630df467 100644--- a/datafusion/spark/src/function/datetime/make_dt_interval.rs+++ b/datafusion/spark/src/function/datetime/make_dt_interval.rs@@ -22,12 +22,14 @@ use arrow::array::{Array, ArrayRef, AsArray, DurationMicrosecondBuilder, PrimitiveArray,};use arrow::datatypes::TimeUnit::Microsecond;-use arrow::datatypes::{DataType, Float64Type, Int32Type};+use arrow::datatypes::{DataType, Field, FieldRef, Float64Type, Int32Type};use datafusion_common::types::{logical_float64, logical_int32, NativeType};-use datafusion_common::{plan_datafusion_err, DataFusionError, Result, ScalarValue};+use datafusion_common::{+ internal_err, plan_datafusion_err, DataFusionError, Result, ScalarValue,+};use datafusion_expr::{- Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,- TypeSignatureClass, Volatility,+ Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,+ Signature, TypeSignature, TypeSignatureClass, Volatility,};use datafusion_functions::utils::make_scalar_function;@@ -99,7 +101,28 @@ impl ScalarUDFImpl for SparkMakeDtInterval {////// [Sail compatibility doc]: https://github.com/lakehq/sail/blob/dc5368daa24d40a7758a299e1ba8fc985cb29108/docs/guide/dataframe/data-types/compatibility.md?plain=1#L260fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {- Ok(DataType::Duration(Microsecond))+ internal_err!("return_field_from_args should be used instead")+ }++ fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {+ let has_non_finite_secs = args+ .scalar_arguments+ .get(3)+ .and_then(|arg| {+ arg.map(|scalar| match scalar {+ ScalarValue::Float64(Some(v)) => !v.is_finite(),+ ScalarValue::Float32(Some(v)) => !v.is_finite(),+ _ => false,+ })+ })+ .unwrap_or(false);+ let nullable =+ has_non_finite_secs || args.arg_fields.iter().any(|f| f.is_nullable());+ Ok(Arc::new(Field::new(+ self.name(),+ DataType::Duration(Microsecond),+ nullable,+ )))}fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {@@ -225,10 +248,9 @@ mod tests {use arrow::array::{DurationMicrosecondArray, Float64Array, Int32Array};use arrow::datatypes::DataType::Duration;- use arrow::datatypes::Field;- use arrow::datatypes::TimeUnit::Microsecond;+ use arrow::datatypes::{DataType, Field, TimeUnit::Microsecond};use datafusion_common::{internal_datafusion_err, DataFusionError, Result};- use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};+ use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};use super::*;@@ -292,6 +314,59 @@ mod tests {Ok(())}+ #[test]+ fn return_field_respects_nullability() -> Result<()> {+ let udf = SparkMakeDtInterval::new();++ // All nullable inputs -> nullable output+ let arg_fields = vec![+ Arc::new(Field::new("days", DataType::Int32, true)),+ Arc::new(Field::new("hours", DataType::Int32, true)),+ Arc::new(Field::new("mins", DataType::Int32, true)),+ Arc::new(Field::new("secs", DataType::Float64, true)),+ ];++ let out = udf.return_field_from_args(ReturnFieldArgs {+ arg_fields: &arg_fields,+ scalar_arguments: &[None, None, None, None],+ })?;+ assert!(out.is_nullable());+ assert_eq!(out.data_type(), &Duration(Microsecond));++ // Non-nullable inputs -> non-nullable output+ let non_nullable_arg_fields = vec![+ Arc::new(Field::new("days", DataType::Int32, false)),+ Arc::new(Field::new("hours", DataType::Int32, false)),+ Arc::new(Field::new("mins", DataType::Int32, false)),+ Arc::new(Field::new("secs", DataType::Float64, false)),+ ];++ let out = udf.return_field_from_args(ReturnFieldArgs {+ arg_fields: &non_nullable_arg_fields,+ scalar_arguments: &[None, None, None, None],+ })?;+ assert!(!out.is_nullable());++ // Non-finite secs scalar should force nullable even if fields are non-nullable+ let scalar_values =+ [None, None, None, Some(ScalarValue::Float64(Some(f64::NAN)))];+ let scalar_refs = scalar_values.iter().map(|v| v.as_ref()).collect::<Vec<_>>();+ let out = udf.return_field_from_args(ReturnFieldArgs {+ arg_fields: &non_nullable_arg_fields,+ scalar_arguments: &scalar_refs,+ })?;+ assert!(out.is_nullable());++ // Zero-arg call (defaults) should also be non-nullable+ let out = udf.return_field_from_args(ReturnFieldArgs {+ arg_fields: &[],+ scalar_arguments: &[],+ })?;+ assert!(!out.is_nullable());++ Ok(())+ }+#[test]fn error_months_overflow_should_be_null() -> Result<()> {// months = year*12 + month → NULL