Apache DataFusion SQL Query Engine
fix(spark): derive concat_ws nullability from separator (#26205)
Vova Kot454ca08b43fd
diff --git a/datafusion/spark/src/function/string/concat_ws.rs b/datafusion/spark/src/function/string/concat_ws.rsindex c9ed1369a5..77e25bbe3b 100644--- a/datafusion/spark/src/function/string/concat_ws.rs+++ b/datafusion/spark/src/function/string/concat_ws.rs@@ -34,9 +34,11 @@ use arrow::array::{StringArray, StringBuilder, StringViewArray,};use arrow::datatypes::{DataType, Field};+use datafusion_common::arrow::datatypes::FieldRef;use datafusion_common::{Result, ScalarValue};use datafusion_expr::{- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ Volatility,};use crate::function::error_utils::{@@ -75,6 +77,17 @@ impl ScalarUDFImpl for SparkConcatWs {Ok(DataType::Utf8)}+ fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result<FieldRef> {+ // Spark skips null values after the separator, so only the separator controls nullability:+ // https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala#L94-L96+ let separator = &args.arg_fields[0];+ Ok(Arc::new(Field::new(+ self.name(),+ DataType::Utf8,+ separator.is_nullable(),+ )))+ }+fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {if arg_types.is_empty() {return Err(invalid_arg_count_exec_err("concat_ws", (1, i32::MAX), 0));diff --git a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt b/datafusion/sqllogictest/test_files/spark/string/concat_ws.sltindex f40408447f..94c6ea0265 100644--- a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt+++ b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt@@ -53,6 +53,11 @@ SELECT concat_ws(',');----(empty)+query TTT+DESCRIBE SELECT concat_ws(',') AS result;+----+result Utf8 NO+## Null separator returns nullquery TSELECT concat_ws(NULL, 'a', 'b', 'c');@@ -113,6 +118,12 @@ SELECT concat_ws(sep) AS result FROM VALUES (CAST(NULL AS STRING)), (',') AS t(sNULL(empty)+query TTT+DESCRIBE SELECT concat_ws(sep) AS result+FROM VALUES (CAST(NULL AS STRING)), (',') AS t(sep);+----+result Utf8 YES+## ── Column expressions ──────────────────────────────────────## concat_ws on columns@@ -130,6 +141,12 @@ a,bcd+query TTT+DESCRIBE SELECT concat_ws(',', a, b) AS result+FROM VALUES ('a', 'b'), ('c', CAST(NULL AS STRING)), (CAST(NULL AS STRING), 'd') AS t(a, b);+----+result Utf8 NO+## Scalar-only arguments over multiple rows (broadcast test)query TSELECT concat_ws(',', 'a', 'b') AS result FROM VALUES (1), (2), (3) AS t(x);