Apache DataFusion SQL Query Engine
feat(spark): implement `StringView` for `SparkConcat` (#19984)
Aryan Anandf5709e7484d3
diff --git a/datafusion/spark/src/function/string/concat.rs b/datafusion/spark/src/function/string/concat.rsindex f3dae22866..d280b9c1c0 100644--- a/datafusion/spark/src/function/string/concat.rs+++ b/datafusion/spark/src/function/string/concat.rs@@ -89,10 +89,21 @@ impl ScalarUDFImpl for SparkConcat {)}fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result<FieldRef> {+ use DataType::*;+// Spark semantics: concat returns NULL if ANY input is NULLlet nullable = args.arg_fields.iter().any(|f| f.is_nullable());- Ok(Arc::new(Field::new("concat", DataType::Utf8, nullable)))+ // Determine return type: Utf8View > LargeUtf8 > Utf8+ let mut dt = &Utf8;+ for field in args.arg_fields {+ let data_type = field.data_type();+ if data_type == &Utf8View || (data_type == &LargeUtf8 && dt != &Utf8View) {+ dt = data_type;+ }+ }++ Ok(Arc::new(Field::new("concat", dt.clone(), nullable)))}}@@ -110,9 +121,18 @@ fn spark_concat(args: ScalarFunctionArgs) -> Result<ColumnarValue> {// Handle zero-argument case: return empty stringif arg_values.is_empty() {- return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(- Some(String::new()),- )));+ let return_type = return_field.data_type();+ return match return_type {+ DataType::Utf8View => Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(+ String::new(),+ )))),+ DataType::LargeUtf8 => Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(+ Some(String::new()),+ ))),+ _ => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(+ Some(String::new()),+ ))),+ };}// Step 1: Check for NULL mask in incoming args@@ -120,7 +140,14 @@ fn spark_concat(args: ScalarFunctionArgs) -> Result<ColumnarValue> {// If all scalars and any is NULL, return NULL immediatelyif matches!(null_mask, NullMaskResolution::ReturnNull) {- return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)));+ let return_type = return_field.data_type();+ return match return_type {+ DataType::Utf8View => Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(None))),+ DataType::LargeUtf8 => {+ Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(None)))+ }+ _ => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))),+ };}// Step 2: Delegate to DataFusion's concat@@ -181,6 +208,7 @@ mod tests {);Ok(())}+#[test]fn test_spark_concat_return_field_non_nullable() -> Result<()> {let func = SparkConcat::new();diff --git a/datafusion/sqllogictest/test_files/spark/string/concat.slt b/datafusion/sqllogictest/test_files/spark/string/concat.sltindex 258cb829d7..97e7b57f7d 100644--- a/datafusion/sqllogictest/test_files/spark/string/concat.slt+++ b/datafusion/sqllogictest/test_files/spark/string/concat.slt@@ -20,6 +20,12 @@ SELECT concat('Spark', 'SQL');----SparkSQL+# Test two Utf8View inputs: value and return type+query TT+SELECT concat(arrow_cast('Spark', 'Utf8View'), arrow_cast('SQL', 'Utf8View')), arrow_typeof(concat(arrow_cast('Spark', 'Utf8View'), arrow_cast('SQL', 'Utf8View')));+----+SparkSQL Utf8View+query TSELECT concat('Spark', 'SQL', NULL);----@@ -46,3 +52,21 @@ SELECT concat(a, b, c) from (select 'a' a, 'b' b, 'c' c union all select null a,----abcNULL++# Test mixed types: Utf8View + Utf8+query TT+SELECT concat(arrow_cast('hello', 'Utf8View'), ' world'), arrow_typeof(concat(arrow_cast('hello', 'Utf8View'), ' world'));+----+hello world Utf8View++# Test Utf8 + LargeUtf8 => return type LargeUtf8+query TT+SELECT concat('a', arrow_cast('b', 'LargeUtf8')), arrow_typeof(concat('a', arrow_cast('b', 'LargeUtf8')));+----+ab LargeUtf8++# Test all three types mixed together+query TT+SELECT concat('a', arrow_cast('b', 'LargeUtf8'), arrow_cast('c', 'Utf8View')), arrow_typeof(concat('a', arrow_cast('b', 'LargeUtf8'), arrow_cast('c', 'Utf8View')));+----+abc Utf8View