Skip to content

Apache DataFusion SQL Query Engine

feat(spark): add `concat_ws` with array support (#20928)

David Lópeze5f7af15a6da
diff --git a/datafusion/spark/src/function/string/concat_ws.rs b/datafusion/spark/src/function/string/concat_ws.rs
new file mode 100644
index 0000000000..c9ed1369a5
--- /dev/null
+++ b/datafusion/spark/src/function/string/concat_ws.rs
@@ -0,0 +1,297 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Spark-compatible `concat_ws`: joins strings (and array elements) with a separator.
+//!
+//! Null scalar args and null array elements are skipped; a null separator yields a
+//! null row. Non-string args are coerced to STRING; list args (`List`, `LargeList`,
+//! `ListView`, `LargeListView`, `FixedSizeList`) expand their elements.
+//!
+//! Differences with DataFusion core `concat_ws`:
+//! - Accepts list arguments and expands their elements
+//! - Always returns Utf8 (Spark's `STRING` type)
+//! - Coerces non-string scalars (numbers, booleans, dates, ...) to Utf8
+
+use std::fmt::Write as _;
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayRef, AsArray, GenericListArray, LargeStringArray, OffsetSizeTrait,
+ StringArray, StringBuilder, StringViewArray,
+};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+
+use crate::function::error_utils::{
+ invalid_arg_count_exec_err, unsupported_data_type_exec_err,
+};
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkConcatWs {
+ signature: Signature,
+}
+
+impl Default for SparkConcatWs {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkConcatWs {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkConcatWs {
+ fn name(&self) -> &str {
+ "concat_ws"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if arg_types.is_empty() {
+ return Err(invalid_arg_count_exec_err("concat_ws", (1, i32::MAX), 0));
+ }
+ Ok(arg_types
+ .iter()
+ .enumerate()
+ .map(|(i, dt)| match dt {
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => dt.clone(),
+ // Non-separator list args expand their elements at runtime.
+ // Normalize the list variant so the kernel only sees
+ // List/LargeList, AND force the element type to Utf8 so the
+ // planner inserts a cast for non-string children (Spark
+ // coerces them to STRING the same way it does for scalars).
+ DataType::List(f)
+ | DataType::ListView(f)
+ | DataType::FixedSizeList(f, _)
+ if i > 0 =>
+ {
+ DataType::List(Arc::new(Field::new(
+ f.name(),
+ DataType::Utf8,
+ f.is_nullable(),
+ )))
+ }
+ DataType::LargeList(f) | DataType::LargeListView(f) if i > 0 => {
+ DataType::LargeList(Arc::new(Field::new(
+ f.name(),
+ DataType::Utf8,
+ f.is_nullable(),
+ )))
+ }
+ // Spark casts everything else (numbers, booleans, dates,
+ // binary, null...) to STRING.
+ _ => DataType::Utf8,
+ })
+ .collect())
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ // Only separator provided → empty string (or NULL if separator is null).
+ // Arg-count validation happens in coerce_types at planning time.
+ if args.args.len() == 1 {
+ return only_separator(&args.args[0]);
+ }
+
+ spark_concat_ws(&args.args, args.number_rows)
+ }
+}
+
+fn only_separator(sep: &ColumnarValue) -> Result<ColumnarValue> {
+ match sep {
+ ColumnarValue::Scalar(s) if s.is_null() => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))
+ }
+ ColumnarValue::Scalar(_) => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
+ String::new(),
+ )))),
+ ColumnarValue::Array(arr) => {
+ let mut builder = StringBuilder::with_capacity(arr.len(), 0);
+ for row_idx in 0..arr.len() {
+ if arr.is_null(row_idx) {
+ builder.append_null();
+ } else {
+ builder.append_value("");
+ }
+ }
+ Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+ }
+ }
+}
+
+fn spark_concat_ws(args: &[ColumnarValue], num_rows: usize) -> Result<ColumnarValue> {
+ let arrays = ColumnarValue::values_to_arrays(args)?;
+ let sep_view = StringView::try_new(&arrays[0])?;
+ let arg_views: Vec<ArgView> = arrays[1..]
+ .iter()
+ .map(ArgView::try_new)
+ .collect::<Result<_>>()?;
+
+ let mut builder = StringBuilder::with_capacity(num_rows, num_rows * 16);
+
+ for row_idx in 0..num_rows {
+ if sep_view.is_null(row_idx) {
+ builder.append_null();
+ continue;
+ }
+
+ // Write parts directly into the builder via its `fmt::Write` impl;
+ // `append_value("")` then finalises the row (offset + validity) with
+ // no extra copy from an intermediate `String`.
+ let separator = sep_view.value(row_idx);
+ let mut first = true;
+ for view in &arg_views {
+ view.write_row(row_idx, separator, &mut builder, &mut first)?;
+ }
+ builder.append_value("");
+ }
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish()) as ArrayRef))
+}
+
+/// Typed view over a string array that downcasts once and exposes
+/// per-row access without further dispatch.
+enum StringView<'a> {
+ Utf8(&'a StringArray),
+ LargeUtf8(&'a LargeStringArray),
+ Utf8View(&'a StringViewArray),
+}
+
+impl<'a> StringView<'a> {
+ fn try_new(arr: &'a ArrayRef) -> Result<Self> {
+ match arr.data_type() {
+ DataType::Utf8 => Ok(Self::Utf8(arr.as_string::<i32>())),
+ DataType::LargeUtf8 => Ok(Self::LargeUtf8(arr.as_string::<i64>())),
+ DataType::Utf8View => Ok(Self::Utf8View(arr.as_string_view())),
+ other => Err(unsupported_data_type_exec_err("concat_ws", "STRING", other)),
+ }
+ }
+
+ fn value(&self, idx: usize) -> &str {
+ match self {
+ Self::Utf8(a) => a.value(idx),
+ Self::LargeUtf8(a) => a.value(idx),
+ Self::Utf8View(a) => a.value(idx),
+ }
+ }
+
+ fn is_null(&self, idx: usize) -> bool {
+ match self {
+ Self::Utf8(a) => a.is_null(idx),
+ Self::LargeUtf8(a) => a.is_null(idx),
+ Self::Utf8View(a) => a.is_null(idx),
+ }
+ }
+}
+
+/// Per-argument view: a string array or a list of strings. The downcast
+/// happens once at construction time. `DataType::Null` cannot appear here —
+/// `coerce_types` rewrites it to `Utf8` before invocation.
+enum ArgView<'a> {
+ Str(StringView<'a>),
+ List(&'a GenericListArray<i32>),
+ LargeList(&'a GenericListArray<i64>),
+}
+
+impl<'a> ArgView<'a> {
+ fn try_new(arr: &'a ArrayRef) -> Result<Self> {
+ match arr.data_type() {
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+ Ok(Self::Str(StringView::try_new(arr)?))
+ }
+ DataType::List(_) => Ok(Self::List(arr.as_list::<i32>())),
+ DataType::LargeList(_) => Ok(Self::LargeList(arr.as_list::<i64>())),
+ other => Err(unsupported_data_type_exec_err(
+ "concat_ws",
+ "STRING or ARRAY<STRING>",
+ other,
+ )),
+ }
+ }
+
+ fn write_row(
+ &self,
+ row_idx: usize,
+ sep: &str,
+ builder: &mut StringBuilder,
+ first: &mut bool,
+ ) -> Result<()> {
+ match self {
+ Self::Str(view) => {
+ if !view.is_null(row_idx) {
+ push_part(builder, view.value(row_idx), sep, first);
+ }
+ }
+ Self::List(list) => write_list_row(*list, row_idx, sep, builder, first)?,
+ Self::LargeList(list) => write_list_row(*list, row_idx, sep, builder, first)?,
+ }
+ Ok(())
+ }
+}
+
+fn write_list_row<O: OffsetSizeTrait>(
+ list: &GenericListArray<O>,
+ row_idx: usize,
+ sep: &str,
+ builder: &mut StringBuilder,
+ first: &mut bool,
+) -> Result<()> {
+ if list.is_null(row_idx) {
+ return Ok(());
+ }
+ let values = list.value(row_idx);
+ // An empty array (e.g. `array()`) contributes nothing — Spark renders it
+ // as the empty string, not an error.
+ if values.is_empty() {
+ return Ok(());
+ }
+ let view = StringView::try_new(&values)?;
+ for i in 0..values.len() {
+ if !view.is_null(i) {
+ push_part(builder, view.value(i), sep, first);
+ }
+ }
+ Ok(())
+}
+
+// `StringBuilder::write_str` only does `extend_from_slice` and never errors;
+// the `.expect(..)` is a documentation hint, not a real failure path.
+fn push_part(builder: &mut StringBuilder, part: &str, sep: &str, first: &mut bool) {
+ if !*first {
+ builder
+ .write_str(sep)
+ .expect("StringBuilder::write_str is infallible");
+ }
+ *first = false;
+ builder
+ .write_str(part)
+ .expect("StringBuilder::write_str is infallible");
+}
diff --git a/datafusion/spark/src/function/string/mod.rs b/datafusion/spark/src/function/string/mod.rs
index 9c90ded5f7..bc94c27732 100644
--- a/datafusion/spark/src/function/string/mod.rs
+++ b/datafusion/spark/src/function/string/mod.rs
@@ -19,6 +19,7 @@ pub mod ascii;
pub mod base64;
pub mod char;
pub mod concat;
+pub mod concat_ws;
pub mod elt;
pub mod format_string;
pub mod ilike;
@@ -40,6 +41,7 @@ make_udf_function!(ascii::SparkAscii, ascii);
make_udf_function!(base64::SparkBase64, base64);
make_udf_function!(char::CharFunc, char);
make_udf_function!(concat::SparkConcat, concat);
+make_udf_function!(concat_ws::SparkConcatWs, concat_ws);
make_udf_function!(ilike::SparkILike, ilike);
make_udf_function!(length::SparkLengthFunc, length);
make_udf_function!(elt::SparkElt, elt);
@@ -77,6 +79,11 @@ pub mod expr_fn {
"Concatenates multiple input strings into a single string. Returns NULL if any input is NULL.",
args
));
+ export_functions!((
+ concat_ws,
+ "Concatenates strings with separator. Supports arrays. Null values are skipped.",
+ sep args
+ ));
export_functions!((
elt,
"Returns the n-th input (1-indexed), e.g. returns 2nd input when n is 2. The function returns NULL if the index is 0 or exceeds the length of the array.",
@@ -142,6 +149,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
base64(),
char(),
concat(),
+ concat_ws(),
elt(),
ilike(),
length(),
diff --git a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
index 62df636bba..f6404cab8f 100644
--- a/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/concat_ws.slt
@@ -21,22 +21,367 @@
# For more information, please see:
# https://github.com/apache/datafusion/issues/15914
-## Original Query: SELECT concat_ws(' ', 'Spark', 'SQL');
-## PySpark 3.5.5 Result: {'concat_ws( , Spark, SQL)': 'Spark SQL', 'typeof(concat_ws( , Spark, SQL))': 'string', 'typeof( )': 'string', 'typeof(Spark)': 'string', 'typeof(SQL)': 'string'}
-#query
-#SELECT concat_ws(' '::string, 'Spark'::string, 'SQL'::string);
-
-## Original Query: SELECT concat_ws('/', 'foo', null, 'bar');
-## PySpark 3.5.5 Result: {'concat_ws(/, foo, NULL, bar)': 'foo/bar', 'typeof(concat_ws(/, foo, NULL, bar))': 'string', 'typeof(/)': 'string', 'typeof(foo)': 'string', 'typeof(NULL)': 'void', 'typeof(bar)': 'string'}
-#query
-#SELECT concat_ws('/'::string, 'foo'::string, NULL::void, 'bar'::string);
-
-## Original Query: SELECT concat_ws('s');
-## PySpark 3.5.5 Result: {'concat_ws(s)': '', 'typeof(concat_ws(s))': 'string', 'typeof(s)': 'string'}
-#query
-#SELECT concat_ws('s'::string);
-
-## Original Query: SELECT concat_ws(null, 'Spark', 'SQL');
-## PySpark 3.5.5 Result: {'concat_ws(NULL, Spark, SQL)': None, 'typeof(concat_ws(NULL, Spark, SQL))': 'string', 'typeof(NULL)': 'void', 'typeof(Spark)': 'string', 'typeof(SQL)': 'string'}
-#query
-#SELECT concat_ws(NULL::void, 'Spark'::string, 'SQL'::string);
+## ── Basic scalar usage ──────────────────────────────────────
+
+## Multiple string arguments
+query T
+SELECT concat_ws(',', 'a', 'b', 'c');
+----
+a,b,c
+
+## Space separator
+query T
+SELECT concat_ws(' ', 'Spark', 'SQL');
+----
+Spark SQL
+
+## Slash separator with null skipped
+query T
+SELECT concat_ws('/', 'foo', NULL, 'bar');
+----
+foo/bar
+
+## Single argument after separator
+query T
+SELECT concat_ws(',', 'a');
+----
+a
+
+## No arguments after separator → empty string
+query T
+SELECT concat_ws(',');
+----
+(empty)
+
+## Null separator returns null
+query T
+SELECT concat_ws(NULL, 'a', 'b', 'c');
+----
+NULL
+
+## All null arguments → empty string
+query T
+SELECT concat_ws(',', CAST(NULL AS STRING), CAST(NULL AS STRING));
+----
+(empty)
+
+## ── Array arguments ─────────────────────────────────────────
+
+## Array argument
+query T
+SELECT concat_ws(',', array('a', 'b', 'c'));
+----
+a,b,c
+
+## Array with nulls skipped
+query T
+SELECT concat_ws(',', array('a', NULL, 'c'));
+----
+a,c
+
+## Multiple arrays
+query T
+SELECT concat_ws(',', array('a', 'b'), array('c', 'd'));
+----
+a,b,c,d
+
+## Mixed scalar and array arguments
+query T
+SELECT concat_ws(',', 'x', array('a', 'b'), 'y');
+----
+x,a,b,y
+
+## Null array is skipped
+query T
+SELECT concat_ws(',', 'x', CAST(NULL AS ARRAY<STRING>), 'y');
+----
+x,y
+
+## ── Edge cases ───────────────────────────────────────────────
+
+## Separator column with no value arguments
+query T
+SELECT concat_ws(sep) AS result FROM VALUES (','), ('-') AS t(sep);
+----
+(empty)
+(empty)
+
+## Null separator in column with no value arguments
+query T
+SELECT concat_ws(sep) AS result FROM VALUES (CAST(NULL AS STRING)), (',') AS t(sep);
+----
+NULL
+(empty)
+
+## ── Column expressions ──────────────────────────────────────
+
+## concat_ws on columns
+query T
+SELECT concat_ws('-', a, b) AS result FROM VALUES ('hello', 'world'), ('foo', 'bar') AS t(a, b);
+----
+hello-world
+foo-bar
+
+## concat_ws with null in columns
+query T
+SELECT concat_ws(',', a, b) AS result FROM VALUES ('a', 'b'), ('c', CAST(NULL AS STRING)), (CAST(NULL AS STRING), 'd') AS t(a, b);
+----
+a,b
+c
+d
+
+## Scalar-only arguments over multiple rows (broadcast test)
+query T
+SELECT concat_ws(',', 'a', 'b') AS result FROM VALUES (1), (2), (3) AS t(x);
+----
+a,b
+a,b
+a,b
+
+## ── Additional edge cases ───────────────────────────────────
+
+## Empty separator — values concatenated with nothing between
+query T
+SELECT concat_ws('', 'a', 'b', 'c');
+----
+abc
+
+## Empty-string values are NOT skipped (only NULLs are)
+query T
+SELECT concat_ws(',', '', 'a', '', 'b');
+----
+,a,,b
+
+## Multi-character separator
+query T
+SELECT concat_ws(' - ', 'a', 'b', 'c');
+----
+a - b - c
+
+## Utf8View separator
+query TT
+SELECT concat_ws(arrow_cast(',', 'Utf8View'), 'a', 'b'), arrow_typeof(concat_ws(arrow_cast(',', 'Utf8View'), 'a', 'b'));
+----
+a,b Utf8
+
+## LargeUtf8 separator
+query TT
+SELECT concat_ws(arrow_cast(',', 'LargeUtf8'), 'a', 'b'), arrow_typeof(concat_ws(arrow_cast(',', 'LargeUtf8'), 'a', 'b'));
+----
+a,b Utf8
+
+## Empty array → empty string
+query T
+SELECT concat_ws(',', array());
+----
+(empty)
+
+## Scalar + array + array mix
+query T
+SELECT concat_ws(',', array('a', 'b'), 'c', array('d', 'e'));
+----
+a,b,c,d,e
+
+## All-NULL row mixed with non-NULL rows
+query T
+SELECT concat_ws(',', a, b, c) AS result FROM VALUES
+ ('a', 'b', 'c'),
+ (CAST(NULL AS STRING), 'b', 'c'),
+ ('a', CAST(NULL AS STRING), CAST(NULL AS STRING)),
+ (CAST(NULL AS STRING), CAST(NULL AS STRING), CAST(NULL AS STRING))
+ AS t(a, b, c);
+----
+a,b,c
+b,c
+a
+(empty)
+
+## Separator from column (per-row separator), with NULL rows
+query T
+SELECT concat_ws(sep, a, b) AS result FROM VALUES
+ (',', 'a', 'b'),
+ (CAST(NULL AS STRING), 'a', 'b'),
+ ('|', 'x', 'y')
+ AS t(sep, a, b);
+----
+a,b
+NULL
+x|y
+
+## ── Spark cross-checked extras ──────────────────────────────
+
+## Zero arguments → error (Spark: WRONG_NUM_ARGS)
+query error
+SELECT concat_ws();
+
+## Numeric separator coerced to string
+query T
+SELECT concat_ws(1, 'a', 'b');
+----
+a1b
+
+## Only numeric separator, no values → empty string
+query T
+SELECT concat_ws(123);
+----
+(empty)
+
+## Numeric values coerced to string
+query T
+SELECT concat_ws(',', 1, 2, 3);
+----
+1,2,3
+
+## Float values
+query T
+SELECT concat_ws(',', 1.5, 2.5);
+----
+1.5,2.5
+
+## Boolean values
+query T
+SELECT concat_ws(',', true, false);
+----
+true,false
+
+## Mixed numeric and string
+query T
+SELECT concat_ws(',', CAST(1 AS BIGINT), 'a');
+----
+1,a
+
+## Date values
+query T
+SELECT concat_ws(',', DATE '2024-01-01', 'x');
+----
+2024-01-01,x
+
+## Multi-byte UTF-8 separator
+query T
+SELECT concat_ws('é', 'a', 'b');
+----
+aéb
+
+## Nested concat_ws
+query T
+SELECT concat_ws('|', concat_ws(',', 'a', 'b'), concat_ws(',', 'c', 'd'));
+----
+a,b|c,d
+
+## All-NULL elements in array → empty string
+query T
+SELECT concat_ws(',', array(CAST(NULL AS STRING), CAST(NULL AS STRING), CAST(NULL AS STRING)));
+----
+(empty)
+
+## Empty-string elements in arrays are NOT skipped
+query T
+SELECT concat_ws(',', array(''), array(''));
+----
+,
+
+## Multiple arrays interleaved with scalars and NULLs
+query T
+SELECT concat_ws(',', array('a', 'b'), 'c', CAST(NULL AS STRING), array('d'), '', 'e');
+----
+a,b,c,d,,e
+
+## Long argument list (variadic)
+query T
+SELECT concat_ws('-', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j');
+----
+a-b-c-d-e-f-g-h-i-j
+
+## Long array
+query T
+SELECT concat_ws(',', array('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'));
+----
+a,b,c,d,e,f,g,h,i,j
+
+## All-empty arguments
+query T
+SELECT concat_ws('', '', '', '');
+----
+(empty)
+
+## Empty separator with all NULLs
+query T
+SELECT concat_ws('', CAST(NULL AS STRING), CAST(NULL AS STRING));
+----
+(empty)
+
+## Long string preserved (length sanity)
+query I
+SELECT length(concat_ws(',', repeat('x', 1000), repeat('y', 1000)));
+----
+2001
+
+## ── List variants (FixedSizeList / ListView) ────────────────
+
+## FixedSizeList argument is expanded element-by-element
+query T
+SELECT concat_ws(',', arrow_cast(make_array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)'));
+----
+a,b,c
+
+## ListView argument is expanded element-by-element
+query T
+SELECT concat_ws(',', arrow_cast(make_array('a', 'b'), 'ListView(Utf8)'));
+----
+a,b
+
+## LargeListView argument is expanded element-by-element
+query T
+SELECT concat_ws(',', arrow_cast(make_array('a', 'b'), 'LargeListView(Utf8)'));
+----
+a,b
+
+## ── Binary coercion (Spark casts binary to its UTF-8 view) ──
+
+## Binary argument is coerced to its string representation
+query T
+SELECT concat_ws(',', X'4869');
+----
+Hi
+
+## ── Null-separator-over-rows shape ──────────────────────────
+
+## Null separator on a multi-row column yields NULL on every row
+query T
+SELECT concat_ws(NULL, v) AS result FROM VALUES ('a'), ('b'), ('c') AS t(v) ORDER BY v;
+----
+NULL
+NULL
+NULL
+
+## ── Non-string list elements (planner-inserted element cast) ─
+
+## Array of integers — elements must be cast to STRING
+query T
+SELECT concat_ws(',', array(1, 2, 3));
+----
+1,2,3
+
+## Array of doubles
+query T
+SELECT concat_ws('-', array(1.5, 2.5, 3.5));
+----
+1.5-2.5-3.5
+
+## Array of booleans
+query T
+SELECT concat_ws(',', array(true, false, true));
+----
+true,false,true
+
+## Mixed: string scalar + int array + string scalar
+query T
+SELECT concat_ws(',', 'x', array(1, 2), 'y');
+----
+x,1,2,y
+
+## ── Struct rejection ────────────────────────────────────────
+
+## Struct argument is rejected (not coerced to string)
+query error
+SELECT concat_ws(',', named_struct('a', 1));