Apache DataFusion SQL Query Engine
feat(spark): add `concat_ws` with array support (#20928)
David Lópeze5f7af15a6da
diff --git a/datafusion/spark/src/function/string/concat_ws.rs b/datafusion/spark/src/function/string/concat_ws.rsnew file mode 100644index 0000000000..c9ed1369a5--- /dev/null+++ b/datafusion/spark/src/function/string/concat_ws.rs@@ -0,0 +1,297 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Spark-compatible `concat_ws`: joins strings (and array elements) with a separator.+//!+//! Null scalar args and null array elements are skipped; a null separator yields a+//! null row. Non-string args are coerced to STRING; list args (`List`, `LargeList`,+//! `ListView`, `LargeListView`, `FixedSizeList`) expand their elements.+//!+//! Differences with DataFusion core `concat_ws`:+//! - Accepts list arguments and expands their elements+//! - Always returns Utf8 (Spark's `STRING` type)+//! - Coerces non-string scalars (numbers, booleans, dates, ...) to Utf8++use std::fmt::Write as _;+use std::sync::Arc;++use arrow::array::{+ Array, ArrayRef, AsArray, GenericListArray, LargeStringArray, OffsetSizeTrait,+ StringArray, StringBuilder, StringViewArray,+};+use arrow::datatypes::{DataType, Field};+use datafusion_common::{Result, ScalarValue};+use datafusion_expr::{+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,+};++use crate::function::error_utils::{+ invalid_arg_count_exec_err, unsupported_data_type_exec_err,+};++#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkConcatWs {+ signature: Signature,+}++impl Default for SparkConcatWs {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkConcatWs {+ pub fn new() -> Self {+ Self {+ signature: Signature::user_defined(Volatility::Immutable),+ }+ }+}++impl ScalarUDFImpl for SparkConcatWs {+ fn name(&self) -> &str {+ "concat_ws"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Utf8)+ }++ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {+ if arg_types.is_empty() {+ return Err(invalid_arg_count_exec_err("concat_ws", (1, i32::MAX), 0));+ }+ Ok(arg_types+ .iter()+ .enumerate()+ .map(|(i, dt)| match dt {+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => dt.clone(),+ // Non-separator list args expand their elements at runtime.+ // Normalize the list variant so the kernel only sees+ // List/LargeList, AND force the element type to Utf8 so the+ // planner inserts a cast for non-string children (Spark+ // coerces them to STRING the same way it does for scalars).+ DataType::List(f)+ | DataType::ListView(f)+ | DataType::FixedSizeList(f, _)+ if i > 0 =>+ {+ DataType::List(Arc::new(Field::new(+ f.name(),+ DataType::Utf8,+ f.is_nullable(),+ )))+ }+ DataType::LargeList(f) | DataType::LargeListView(f) if i > 0 => {+ DataType::LargeList(Arc::new(Field::new(+ f.name(),+ DataType::Utf8,+ f.is_nullable(),+ )))+ }+ // Spark casts everything else (numbers, booleans, dates,+ // binary, null...) to STRING.+ _ => DataType::Utf8,+ })+ .collect())+ }++ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {+ // Only separator provided → empty string (or NULL if separator is null).+ // Arg-count validation happens in coerce_types at planning time.+ if args.args.len() == 1 {+ return only_separator(&args.args[0]);+ }++ spark_concat_ws(&args.args, args.number_rows)+ }+}++fn only_separator(sep: &ColumnarValue) -> Result<ColumnarValue> {+ match sep {+ ColumnarValue::Scalar(s) if s.is_null() => {+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))+ }+ ColumnarValue::Scalar(_) => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(+ String::new(),+ )))),+ ColumnarValue::Array(arr) => {+ let mut builder = StringBuilder::with_capacity(arr.len(), 0);+ for row_idx in 0..arr.len() {+ if arr.is_null(row_idx) {+ builder.append_null();+ } else {+ builder.append_value("");+ }+ }+ Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))+ }+ }+}++fn spark_concat_ws(args: &[ColumnarValue], num_rows: usize) -> Result<ColumnarValue> {+ let arrays = ColumnarValue::values_to_arrays(args)?;+ let sep_view = StringView::try_new(&arrays[0])?;+ let arg_views: Vec<ArgView> = arrays[1..]+ .iter()+ .map(ArgView::try_new)+ .collect::<Result<_>>()?;++ let mut builder = StringBuilder::with_capacity(num_rows, num_rows * 16);++ for row_idx in 0..num_rows {+ if sep_view.is_null(row_idx) {+ builder.append_null();+ continue;+ }++ // Write parts directly into the builder via its `fmt::Write` impl;+ // `append_value("")` then finalises the row (offset + validity) with+ // no extra copy from an intermediate `String`.+ let separator = sep_view.value(row_idx);+ let mut first = true;+ for view in &arg_views {+ view.write_row(row_idx, separator, &mut builder, &mut first)?;+ }+ builder.append_value("");+ }++ Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))+}++/// Typed view over a string array that downcasts once and exposes+/// per-row access without further dispatch.+enum StringView<'a> {+ Utf8(&'a StringArray),+ LargeUtf8(&'a LargeStringArray),+ Utf8View(&'a StringViewArray),+}++impl<'a> StringView<'a> {+ fn try_new(arr: &'a ArrayRef) -> Result<Self> {+ match arr.data_type() {+ DataType::Utf8 => Ok(Self::Utf8(arr.as_string::<i32>())),+ DataType::LargeUtf8 => Ok(Self::LargeUtf8(arr.as_string::<i64>())),+ DataType::Utf8View => Ok(Self::Utf8View(arr.as_string_view())),+ other => Err(unsupported_data_type_exec_err("concat_ws", "STRING", other)),+ }+ }++ fn value(&self, idx: usize) -> &str {+ match self {+ Self::Utf8(a) => a.value(idx),+ Self::LargeUtf8(a) => a.value(idx),+ Self::Utf8View(a) => a.value(idx),+ }+ }++ fn is_null(&self, idx: usize) -> bool {+ match self {+ Self::Utf8(a) => a.is_null(idx),+ Self::LargeUtf8(a) => a.is_null(idx),+ Self::Utf8View(a) => a.is_null(idx),+ }+ }+}++/// Per-argument view: a string array or a list of strings. The downcast+/// happens once at construction time. `DataType::Null` cannot appear here —+/// `coerce_types` rewrites it to `Utf8` before invocation.+enum ArgView<'a> {+ Str(StringView<'a>),+ List(&'a GenericListArray<i32>),+ LargeList(&'a GenericListArray<i64>),+}++impl<'a> ArgView<'a> {+ fn try_new(arr: &'a ArrayRef) -> Result<Self> {+ match arr.data_type() {+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {+ Ok(Self::Str(StringView::try_new(arr)?))+ }+ DataType::List(_) => Ok(Self::List(arr.as_list::<i32>())),+ DataType::LargeList(_) => Ok(Self::LargeList(arr.as_list::<i64>())),+ other => Err(unsupported_data_type_exec_err(+ "concat_ws",+ "STRING or ARRAY<STRING>",+ other,+ )),+ }+ }++ fn write_row(+ &self,+ row_idx: usize,+ sep: &str,+ builder: &mut StringBuilder,+ first: &mut bool,+ ) -> Result<()> {+ match self {+ Self::Str(view) => {+ if !view.is_null(row_idx) {+ push_part(builder, view.value(row_idx), sep, first);+ }+ }+ Self::List(list) => write_list_row(*list, row_idx, sep, builder, first)?,+ Self::LargeList(list) => write_list_row(*list, row_idx, sep, builder, first)?,+ }+ Ok(())+ }+}++fn write_list_row<O: OffsetSizeTrait>(+ list: &GenericListArray<O>,+ row_idx: usize,+ sep: &str,+ builder: &mut StringBuilder,+ first: &mut bool,+) -> Result<()> {+ if list.is_null(row_idx) {+ return Ok(());+ }+ let values = list.value(row_idx);+ // An empty array (e.g. `array()`) contributes nothing — Spark renders it+ // as the empty string, not an error.+ if values.is_empty() {+ return Ok(());+ }+ let view = StringView::try_new(&values)?;+ for i in 0..values.len() {+ if !view.is_null(i) {+ push_part(builder, view.value(i), sep, first);+ }+ }+ Ok(())+}++// `StringBuilder::write_str` only does `extend_from_slice` and never errors;+// the `.expect(..)` is a documentation hint, not a real failure path.+fn push_part(builder: &mut StringBuilder, part: &str, sep: &str, first: &mut bool) {+ if !*first {+ builder+ .write_str(sep)+ .expect("StringBuilder::write_str is infallible");+ }+ *first = false;+ builder+ .write_str(part)+ .expect("StringBuilder::write_str is infallible");+}diff --git a/datafusion/spark/src/function/string/mod.rs b/datafusion/spark/src/function/string/mod.rsindex 9c90ded5f7..bc94c27732 100644--- a/datafusion/spark/src/function/string/mod.rs+++ b/datafusion/spark/src/function/string/mod.rs@@ -19,6 +19,7 @@ pub mod ascii;pub mod base64;pub mod char;pub mod concat;+pub mod concat_ws;pub mod elt;pub mod format_string;pub mod ilike;@@ -40,6 +41,7 @@ make_udf_function!(ascii::SparkAscii, ascii);make_udf_function!(base64::SparkBase64, base64);make_udf_function!(char::CharFunc, char);make_udf_function!(concat::SparkConcat, concat);+make_udf_function!(concat_ws::SparkConcatWs, concat_ws);make_udf_function!(ilike::SparkILike, ilike);make_udf_function!(length::SparkLengthFunc, length);make_udf_function!(elt::SparkElt, elt);@@ -77,6 +79,11 @@ pub mod expr_fn {"Concatenates multiple input strings into a single string. Returns NULL if any input is NULL.",args));+ export_functions!((+ concat_ws,+ "Concatenates strings with separator. Supports arrays. Null values are skipped.",+ sep args+ ));export_functions!((elt,"Returns the n-th input (1-indexed), e.g. returns 2nd input when n is 2. The function returns NULL if the index is 0 or exceeds the length of the array.",@@ -142,6 +149,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {base64(),char(),concat(),+ concat_ws(),elt(),ilike(),length(),diff --git a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt b/datafusion/sqllogictest/test_files/spark/string/concat_ws.sltindex 62df636bba..f6404cab8f 100644--- a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt+++ b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt@@ -21,22 +21,367 @@# For more information, please see:# https://github.com/apache/datafusion/issues/15914-## Original Query: SELECT concat_ws(' ', 'Spark', 'SQL');-## PySpark 3.5.5 Result: {'concat_ws( , Spark, SQL)': 'Spark SQL', 'typeof(concat_ws( , Spark, SQL))': 'string', 'typeof( )': 'string', 'typeof(Spark)': 'string', 'typeof(SQL)': 'string'}-#query-#SELECT concat_ws(' '::string, 'Spark'::string, 'SQL'::string);--## Original Query: SELECT concat_ws('/', 'foo', null, 'bar');-## PySpark 3.5.5 Result: {'concat_ws(/, foo, NULL, bar)': 'foo/bar', 'typeof(concat_ws(/, foo, NULL, bar))': 'string', 'typeof(/)': 'string', 'typeof(foo)': 'string', 'typeof(NULL)': 'void', 'typeof(bar)': 'string'}-#query-#SELECT concat_ws('/'::string, 'foo'::string, NULL::void, 'bar'::string);--## Original Query: SELECT concat_ws('s');-## PySpark 3.5.5 Result: {'concat_ws(s)': '', 'typeof(concat_ws(s))': 'string', 'typeof(s)': 'string'}-#query-#SELECT concat_ws('s'::string);--## Original Query: SELECT concat_ws(null, 'Spark', 'SQL');-## PySpark 3.5.5 Result: {'concat_ws(NULL, Spark, SQL)': None, 'typeof(concat_ws(NULL, Spark, SQL))': 'string', 'typeof(NULL)': 'void', 'typeof(Spark)': 'string', 'typeof(SQL)': 'string'}-#query-#SELECT concat_ws(NULL::void, 'Spark'::string, 'SQL'::string);+## ── Basic scalar usage ──────────────────────────────────────++## Multiple string arguments+query T+SELECT concat_ws(',', 'a', 'b', 'c');+----+a,b,c++## Space separator+query T+SELECT concat_ws(' ', 'Spark', 'SQL');+----+Spark SQL++## Slash separator with null skipped+query T+SELECT concat_ws('/', 'foo', NULL, 'bar');+----+foo/bar++## Single argument after separator+query T+SELECT concat_ws(',', 'a');+----+a++## No arguments after separator → empty string+query T+SELECT concat_ws(',');+----+(empty)++## Null separator returns null+query T+SELECT concat_ws(NULL, 'a', 'b', 'c');+----+NULL++## All null arguments → empty string+query T+SELECT concat_ws(',', CAST(NULL AS STRING), CAST(NULL AS STRING));+----+(empty)++## ── Array arguments ─────────────────────────────────────────++## Array argument+query T+SELECT concat_ws(',', array('a', 'b', 'c'));+----+a,b,c++## Array with nulls skipped+query T+SELECT concat_ws(',', array('a', NULL, 'c'));+----+a,c++## Multiple arrays+query T+SELECT concat_ws(',', array('a', 'b'), array('c', 'd'));+----+a,b,c,d++## Mixed scalar and array arguments+query T+SELECT concat_ws(',', 'x', array('a', 'b'), 'y');+----+x,a,b,y++## Null array is skipped+query T+SELECT concat_ws(',', 'x', CAST(NULL AS ARRAY<STRING>), 'y');+----+x,y++## ── Edge cases ───────────────────────────────────────────────++## Separator column with no value arguments+query T+SELECT concat_ws(sep) AS result FROM VALUES (','), ('-') AS t(sep);+----+(empty)+(empty)++## Null separator in column with no value arguments+query T+SELECT concat_ws(sep) AS result FROM VALUES (CAST(NULL AS STRING)), (',') AS t(sep);+----+NULL+(empty)++## ── Column expressions ──────────────────────────────────────++## concat_ws on columns+query T+SELECT concat_ws('-', a, b) AS result FROM VALUES ('hello', 'world'), ('foo', 'bar') AS t(a, b);+----+hello-world+foo-bar++## concat_ws with null in columns+query T+SELECT concat_ws(',', a, b) AS result FROM VALUES ('a', 'b'), ('c', CAST(NULL AS STRING)), (CAST(NULL AS STRING), 'd') AS t(a, b);+----+a,b+c+d++## Scalar-only arguments over multiple rows (broadcast test)+query T+SELECT concat_ws(',', 'a', 'b') AS result FROM VALUES (1), (2), (3) AS t(x);+----+a,b+a,b+a,b++## ── Additional edge cases ───────────────────────────────────++## Empty separator — values concatenated with nothing between+query T+SELECT concat_ws('', 'a', 'b', 'c');+----+abc++## Empty-string values are NOT skipped (only NULLs are)+query T+SELECT concat_ws(',', '', 'a', '', 'b');+----+,a,,b++## Multi-character separator+query T+SELECT concat_ws(' - ', 'a', 'b', 'c');+----+a - b - c++## Utf8View separator+query TT+SELECT concat_ws(arrow_cast(',', 'Utf8View'), 'a', 'b'), arrow_typeof(concat_ws(arrow_cast(',', 'Utf8View'), 'a', 'b'));+----+a,b Utf8++## LargeUtf8 separator+query TT+SELECT concat_ws(arrow_cast(',', 'LargeUtf8'), 'a', 'b'), arrow_typeof(concat_ws(arrow_cast(',', 'LargeUtf8'), 'a', 'b'));+----+a,b Utf8++## Empty array → empty string+query T+SELECT concat_ws(',', array());+----+(empty)++## Scalar + array + array mix+query T+SELECT concat_ws(',', array('a', 'b'), 'c', array('d', 'e'));+----+a,b,c,d,e++## All-NULL row mixed with non-NULL rows+query T+SELECT concat_ws(',', a, b, c) AS result FROM VALUES+ ('a', 'b', 'c'),+ (CAST(NULL AS STRING), 'b', 'c'),+ ('a', CAST(NULL AS STRING), CAST(NULL AS STRING)),+ (CAST(NULL AS STRING), CAST(NULL AS STRING), CAST(NULL AS STRING))+ AS t(a, b, c);+----+a,b,c+b,c+a+(empty)++## Separator from column (per-row separator), with NULL rows+query T+SELECT concat_ws(sep, a, b) AS result FROM VALUES+ (',', 'a', 'b'),+ (CAST(NULL AS STRING), 'a', 'b'),+ ('|', 'x', 'y')+ AS t(sep, a, b);+----+a,b+NULL+x|y++## ── Spark cross-checked extras ──────────────────────────────++## Zero arguments → error (Spark: WRONG_NUM_ARGS)+query error+SELECT concat_ws();++## Numeric separator coerced to string+query T+SELECT concat_ws(1, 'a', 'b');+----+a1b++## Only numeric separator, no values → empty string+query T+SELECT concat_ws(123);+----+(empty)++## Numeric values coerced to string+query T+SELECT concat_ws(',', 1, 2, 3);+----+1,2,3++## Float values+query T+SELECT concat_ws(',', 1.5, 2.5);+----+1.5,2.5++## Boolean values+query T+SELECT concat_ws(',', true, false);+----+true,false++## Mixed numeric and string+query T+SELECT concat_ws(',', CAST(1 AS BIGINT), 'a');+----+1,a++## Date values+query T+SELECT concat_ws(',', DATE '2024-01-01', 'x');+----+2024-01-01,x++## Multi-byte UTF-8 separator+query T+SELECT concat_ws('é', 'a', 'b');+----+aéb++## Nested concat_ws+query T+SELECT concat_ws('|', concat_ws(',', 'a', 'b'), concat_ws(',', 'c', 'd'));+----+a,b|c,d++## All-NULL elements in array → empty string+query T+SELECT concat_ws(',', array(CAST(NULL AS STRING), CAST(NULL AS STRING), CAST(NULL AS STRING)));+----+(empty)++## Empty-string elements in arrays are NOT skipped+query T+SELECT concat_ws(',', array(''), array(''));+----+,++## Multiple arrays interleaved with scalars and NULLs+query T+SELECT concat_ws(',', array('a', 'b'), 'c', CAST(NULL AS STRING), array('d'), '', 'e');+----+a,b,c,d,,e++## Long argument list (variadic)+query T+SELECT concat_ws('-', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j');+----+a-b-c-d-e-f-g-h-i-j++## Long array+query T+SELECT concat_ws(',', array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'));+----+a,b,c,d,e,f,g,h,i,j++## All-empty arguments+query T+SELECT concat_ws('', '', '', '');+----+(empty)++## Empty separator with all NULLs+query T+SELECT concat_ws('', CAST(NULL AS STRING), CAST(NULL AS STRING));+----+(empty)++## Long string preserved (length sanity)+query I+SELECT length(concat_ws(',', repeat('x', 1000), repeat('y', 1000)));+----+2001++## ── List variants (FixedSizeList / ListView) ────────────────++## FixedSizeList argument is expanded element-by-element+query T+SELECT concat_ws(',', arrow_cast(make_array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)'));+----+a,b,c++## ListView argument is expanded element-by-element+query T+SELECT concat_ws(',', arrow_cast(make_array('a', 'b'), 'ListView(Utf8)'));+----+a,b++## LargeListView argument is expanded element-by-element+query T+SELECT concat_ws(',', arrow_cast(make_array('a', 'b'), 'LargeListView(Utf8)'));+----+a,b++## ── Binary coercion (Spark casts binary to its UTF-8 view) ──++## Binary argument is coerced to its string representation+query T+SELECT concat_ws(',', X'4869');+----+Hi++## ── Null-separator-over-rows shape ──────────────────────────++## Null separator on a multi-row column yields NULL on every row+query T+SELECT concat_ws(NULL, v) AS result FROM VALUES ('a'), ('b'), ('c') AS t(v) ORDER BY v;+----+NULL+NULL+NULL++## ── Non-string list elements (planner-inserted element cast) ─++## Array of integers — elements must be cast to STRING+query T+SELECT concat_ws(',', array(1, 2, 3));+----+1,2,3++## Array of doubles+query T+SELECT concat_ws('-', array(1.5, 2.5, 3.5));+----+1.5-2.5-3.5++## Array of booleans+query T+SELECT concat_ws(',', array(true, false, true));+----+true,false,true++## Mixed: string scalar + int array + string scalar+query T+SELECT concat_ws(',', 'x', array(1, 2), 'y');+----+x,1,2,y++## ── Struct rejection ────────────────────────────────────────++## Struct argument is rejected (not coerced to string)+query error+SELECT concat_ws(',', named_struct('a', 1));