Skip to content

Apache DataFusion SQL Query Engine

fix(concat): correct nullability inference (nullable only if all arguments nullable) (#19189)

Ujjwal Kumar Tiwari3a67d86c943f
diff --git a/datafusion/spark/src/function/string/concat.rs b/datafusion/spark/src/function/string/concat.rs
index 772b280320..8e97e591fc 100644
--- a/datafusion/spark/src/function/string/concat.rs
+++ b/datafusion/spark/src/function/string/concat.rs
@@ -17,8 +17,10 @@
use arrow::array::Array;
use arrow::buffer::NullBuffer;
-use arrow::datatypes::DataType;
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::arrow::datatypes::FieldRef;
use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::ReturnFieldArgs;
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
@@ -71,10 +73,6 @@ impl ScalarUDFImpl for SparkConcat {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Utf8)
- }
-
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
spark_concat(args)
}
@@ -83,6 +81,17 @@ impl ScalarUDFImpl for SparkConcat {
// Accept any string types, including zero arguments
Ok(arg_types.to_vec())
}
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ datafusion_common::internal_err!(
+ "return_type should not be called for Spark concat"
+ )
+ }
+ fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result<FieldRef> {
+ // Spark semantics: concat returns NULL if ANY input is NULL
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+
+ Ok(Arc::new(Field::new("concat", DataType::Utf8, nullable)))
+ }
}
/// Represents the null state for Spark concat
@@ -231,8 +240,10 @@ mod tests {
use super::*;
use crate::function::utils::test::test_scalar_function;
use arrow::array::StringArray;
- use arrow::datatypes::DataType;
+ use arrow::datatypes::{DataType, Field};
use datafusion_common::Result;
+ use datafusion_expr::ReturnFieldArgs;
+ use std::sync::Arc;
#[test]
fn test_concat_basic() -> Result<()> {
@@ -266,4 +277,50 @@ mod tests {
);
Ok(())
}
+ #[test]
+ fn test_spark_concat_return_field_non_nullable() -> Result<()> {
+ let func = SparkConcat::new();
+
+ let fields = vec![
+ Arc::new(Field::new("a", DataType::Utf8, false)),
+ Arc::new(Field::new("b", DataType::Utf8, false)),
+ ];
+
+ let args = ReturnFieldArgs {
+ arg_fields: &fields,
+ scalar_arguments: &[],
+ };
+
+ let field = func.return_field_from_args(args)?;
+
+ assert!(
+ !field.is_nullable(),
+ "Expected concat result to be non-nullable when all inputs are non-nullable"
+ );
+
+ Ok(())
+ }
+ #[test]
+ fn test_spark_concat_return_field_nullable() -> Result<()> {
+ let func = SparkConcat::new();
+
+ let fields = vec![
+ Arc::new(Field::new("a", DataType::Utf8, false)),
+ Arc::new(Field::new("b", DataType::Utf8, true)),
+ ];
+
+ let args = ReturnFieldArgs {
+ arg_fields: &fields,
+ scalar_arguments: &[],
+ };
+
+ let field = func.return_field_from_args(args)?;
+
+ assert!(
+ field.is_nullable(),
+ "Expected concat result to be nullable when any input is nullable"
+ );
+
+ Ok(())
+ }
}