Skip to content

Apache DataFusion SQL Query Engine

fix(spark): derive concat_ws nullability from separator (#26205)

Vova Kot454ca08b43fd
diff --git a/datafusion/spark/src/function/string/concat_ws.rs b/datafusion/spark/src/function/string/concat_ws.rs
index c9ed1369a5..77e25bbe3b 100644
--- a/datafusion/spark/src/function/string/concat_ws.rs
+++ b/datafusion/spark/src/function/string/concat_ws.rs
@@ -34,9 +34,11 @@ use arrow::array::{
StringArray, StringBuilder, StringViewArray,
};
use arrow::datatypes::{DataType, Field};
+use datafusion_common::arrow::datatypes::FieldRef;
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
};
use crate::function::error_utils::{
@@ -75,6 +77,17 @@ impl ScalarUDFImpl for SparkConcatWs {
Ok(DataType::Utf8)
}
+ fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result<FieldRef> {
+ // Spark skips null values after the separator, so only the separator controls nullability:
+ // https://github.com/apache/spark/blob/32f7299601108917fb01920a54e084595b7b3bf8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala#L94-L96
+ let separator = &args.arg_fields[0];
+ Ok(Arc::new(Field::new(
+ self.name(),
+ DataType::Utf8,
+ separator.is_nullable(),
+ )))
+ }
+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
if arg_types.is_empty() {
return Err(invalid_arg_count_exec_err("concat_ws", (1, i32::MAX), 0));
diff --git a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
index f40408447f..94c6ea0265 100644
--- a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
@@ -53,6 +53,11 @@ SELECT concat_ws(',');
----
(empty)
+query TTT
+DESCRIBE SELECT concat_ws(',') AS result;
+----
+result Utf8 NO
+
## Null separator returns null
query T
SELECT concat_ws(NULL, 'a', 'b', 'c');
@@ -113,6 +118,12 @@ SELECT concat_ws(sep) AS result FROM VALUES (CAST(NULL AS STRING)), (',') AS t(s
NULL
(empty)
+query TTT
+DESCRIBE SELECT concat_ws(sep) AS result
+FROM VALUES (CAST(NULL AS STRING)), (',') AS t(sep);
+----
+result Utf8 YES
+
## ── Column expressions ──────────────────────────────────────
## concat_ws on columns
@@ -130,6 +141,12 @@ a,b
c
d
+query TTT
+DESCRIBE SELECT concat_ws(',', a, b) AS result
+FROM VALUES ('a', 'b'), ('c', CAST(NULL AS STRING)), (CAST(NULL AS STRING), 'd') AS t(a, b);
+----
+result Utf8 NO
+
## Scalar-only arguments over multiple rows (broadcast test)
query T
SELECT concat_ws(',', 'a', 'b') AS result FROM VALUES (1), (2), (3) AS t(x);