Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize `string_to_array` for scalar args (#21131)

Neil Conwayd87d8f6cacc0
diff --git a/Cargo.lock b/Cargo.lock
index e0cc5845c0..802bfc4703 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2285,6 +2285,7 @@ dependencies = [
"itertools 0.14.0",
"itoa",
"log",
+ "memchr",
"rand 0.9.2",
]
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index 6e96a44fc9..31462d5e50 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -61,6 +61,7 @@ hashbrown = { workspace = true }
itertools = { workspace = true, features = ["use_std"] }
itoa = { workspace = true }
log = { workspace = true }
+memchr = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["async_tokio"] }
@@ -117,3 +118,7 @@ name = "array_position"
[[bench]]
harness = false
name = "array_sort"
+
+[[bench]]
+harness = false
+name = "string_to_array"
diff --git a/datafusion/functions-nested/benches/string_to_array.rs b/datafusion/functions-nested/benches/string_to_array.rs
new file mode 100644
index 0000000000..e403d5e51b
--- /dev/null
+++ b/datafusion/functions-nested/benches/string_to_array.rs
@@ -0,0 +1,244 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{ArrayRef, StringArray};
+use arrow::datatypes::{DataType, Field};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+use datafusion_common::ScalarValue;
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_functions_nested::string::StringToArray;
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::hint::black_box;
+use std::sync::Arc;
+
+const NUM_ROWS: usize = 1000;
+const SEED: u64 = 42;
+
+fn criterion_benchmark(c: &mut Criterion) {
+ // Single-char delimiter
+ let comma = ColumnarValue::Scalar(ScalarValue::Utf8(Some(",".to_string())));
+ bench_string_to_array(
+ c,
+ "string_to_array_single_char_delim",
+ create_csv_strings,
+ &comma,
+ None,
+ );
+
+ // Multi-char delimiter
+ let double_colon = ColumnarValue::Scalar(ScalarValue::Utf8(Some("::".to_string())));
+ bench_string_to_array(
+ c,
+ "string_to_array_multi_char_delim",
+ create_multi_delim_strings,
+ &double_colon,
+ None,
+ );
+
+ // With null_str argument
+ let null_str = ColumnarValue::Scalar(ScalarValue::Utf8(Some("NULL".to_string())));
+ bench_string_to_array(
+ c,
+ "string_to_array_with_null_str",
+ create_csv_strings_with_nulls,
+ &comma,
+ Some(&null_str),
+ );
+
+ // NULL delimiter
+ let null_delim = ColumnarValue::Scalar(ScalarValue::Utf8(None));
+ bench_string_to_array(
+ c,
+ "string_to_array_null_delim",
+ create_short_strings,
+ &null_delim,
+ None,
+ );
+
+ // Columnar delimiter (fall-back path)
+ bench_string_to_array_columnar_delim(c);
+}
+
+fn bench_string_to_array_columnar_delim(c: &mut Criterion) {
+ let mut group = c.benchmark_group("string_to_array_columnar_delim");
+
+ for &num_elements in &[5, 20, 100] {
+ let string_array = create_csv_strings(num_elements);
+ let delimiter_array: ArrayRef =
+ Arc::new(StringArray::from(vec![Some(","); NUM_ROWS]));
+
+ let args = vec![
+ ColumnarValue::Array(string_array.clone()),
+ ColumnarValue::Array(delimiter_array),
+ ];
+ let arg_fields = vec![
+ Field::new("str", DataType::Utf8, true).into(),
+ Field::new("delimiter", DataType::Utf8, false).into(),
+ ];
+
+ let return_field = Field::new(
+ "result",
+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),
+ true,
+ );
+
+ group.bench_with_input(
+ BenchmarkId::from_parameter(num_elements),
+ &num_elements,
+ |b, _| {
+ let udf = StringToArray::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone().into(),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+
+ group.finish();
+}
+
+fn bench_string_to_array(
+ c: &mut Criterion,
+ group_name: &str,
+ make_strings: fn(usize) -> ArrayRef,
+ delimiter: &ColumnarValue,
+ null_str: Option<&ColumnarValue>,
+) {
+ let mut group = c.benchmark_group(group_name);
+
+ for &num_elements in &[5, 20, 100] {
+ let string_array = make_strings(num_elements);
+
+ let mut args = vec![
+ ColumnarValue::Array(string_array.clone()),
+ delimiter.clone(),
+ ];
+ let mut arg_fields = vec![
+ Field::new("str", DataType::Utf8, true).into(),
+ Field::new("delimiter", DataType::Utf8, true).into(),
+ ];
+ if let Some(ns) = null_str {
+ args.push(ns.clone());
+ arg_fields.push(Field::new("null_str", DataType::Utf8, true).into());
+ }
+
+ let return_field = Field::new(
+ "result",
+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))),
+ true,
+ );
+
+ group.bench_with_input(
+ BenchmarkId::from_parameter(num_elements),
+ &num_elements,
+ |b, _| {
+ let udf = StringToArray::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: NUM_ROWS,
+ return_field: return_field.clone().into(),
+ config_options: Arc::new(ConfigOptions::default()),
+ })
+ .unwrap(),
+ )
+ })
+ },
+ );
+ }
+
+ group.finish();
+}
+
+/// Creates strings like "val1,val2,val3,...,valN" with `num_elements` elements.
+fn create_csv_strings(num_elements: usize) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let strings: StringArray = (0..NUM_ROWS)
+ .map(|_| {
+ let parts: Vec<String> = (0..num_elements)
+ .map(|_| format!("val{}", rng.random_range(0..1000)))
+ .collect();
+ Some(parts.join(","))
+ })
+ .collect();
+ Arc::new(strings)
+}
+
+/// Creates strings like "val1::val2::val3::...::valN".
+fn create_multi_delim_strings(num_elements: usize) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let strings: StringArray = (0..NUM_ROWS)
+ .map(|_| {
+ let parts: Vec<String> = (0..num_elements)
+ .map(|_| format!("val{}", rng.random_range(0..1000)))
+ .collect();
+ Some(parts.join("::"))
+ })
+ .collect();
+ Arc::new(strings)
+}
+
+/// Creates CSV strings where ~10% of elements are the literal "NULL".
+fn create_csv_strings_with_nulls(num_elements: usize) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let strings: StringArray = (0..NUM_ROWS)
+ .map(|_| {
+ let parts: Vec<String> = (0..num_elements)
+ .map(|_| {
+ if rng.random::<f64>() < 0.1 {
+ "NULL".to_string()
+ } else {
+ format!("val{}", rng.random_range(0..1000))
+ }
+ })
+ .collect();
+ Some(parts.join(","))
+ })
+ .collect();
+ Arc::new(strings)
+}
+
+/// Creates short strings (length = `num_chars`) for the NULL-delimiter
+/// (split-into-characters) benchmark.
+fn create_short_strings(num_chars: usize) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let strings: StringArray = (0..NUM_ROWS)
+ .map(|_| {
+ let s: String = (0..num_chars)
+ .map(|_| rng.random_range(b'a'..=b'z') as char)
+ .collect();
+ Some(s)
+ })
+ .collect();
+ Arc::new(strings)
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions-nested/src/string.rs b/datafusion/functions-nested/src/string.rs
index aa2ae69bea..b76736672c 100644
--- a/datafusion/functions-nested/src/string.rs
+++ b/datafusion/functions-nested/src/string.rs
@@ -26,13 +26,13 @@ use arrow::array::{
use arrow::datatypes::{DataType, Field};
use datafusion_common::utils::ListCoercion;
-use datafusion_common::{DataFusionError, Result, not_impl_err};
+use datafusion_common::{DataFusionError, Result, ScalarValue, not_impl_err};
use std::fmt::{self, Write};
use crate::utils::make_scalar_function;
use arrow::array::{
- GenericStringArray, StringArrayType, StringViewArray,
+ StringArrayType, StringViewArray,
builder::{ArrayBuilder, LargeStringBuilder, StringViewBuilder},
cast::AsArray,
};
@@ -43,8 +43,8 @@ use arrow::datatypes::DataType::{
use datafusion_common::cast::{
as_fixed_size_list_array, as_large_list_array, as_list_array,
};
-use datafusion_common::exec_err;
use datafusion_common::types::logical_string;
+use datafusion_common::{exec_datafusion_err, exec_err};
use datafusion_expr::{
ArrayFunctionArgument, ArrayFunctionSignature, Coercion, ColumnarValue,
Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
@@ -189,11 +189,17 @@ make_udf_expr_and_func!(
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
-pub(super) struct StringToArray {
+pub struct StringToArray {
signature: Signature,
aliases: Vec<String>,
}
+impl Default for StringToArray {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
impl StringToArray {
pub fn new() -> Self {
Self {
@@ -233,13 +239,71 @@ impl ScalarUDFImpl for StringToArray {
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- let args = &args.args;
- match args[0].data_type() {
- Utf8 | Utf8View => make_scalar_function(string_to_array_inner::<i32>)(args),
- LargeUtf8 => make_scalar_function(string_to_array_inner::<i64>)(args),
+ let ScalarFunctionArgs { args, .. } = args;
+
+ let delimiter_is_scalar = matches!(&args[1], ColumnarValue::Scalar(_));
+ let null_str_is_scalar = args
+ .get(2)
+ .is_none_or(|a| matches!(a, ColumnarValue::Scalar(_)));
+
+ if !(delimiter_is_scalar && null_str_is_scalar) {
+ return make_scalar_function(string_to_array_fallback)(&args);
+ }
+
+ // Delimiter and null_str (if given) are scalar, so use the fast path
+ let delimiter = match &args[1] {
+ ColumnarValue::Scalar(s) => s.try_as_str().ok_or_else(|| {
+ exec_datafusion_err!(
+ "unsupported type for string_to_array delimiter: {:?}",
+ args[1].data_type()
+ )
+ })?,
+ _ => unreachable!("delimiter must be scalar in this branch"),
+ };
+ let null_value = match args.get(2) {
+ Some(ColumnarValue::Scalar(s)) => s.try_as_str().ok_or_else(|| {
+ exec_datafusion_err!(
+ "unsupported type for string_to_array null_str: {:?}",
+ args[2].data_type()
+ )
+ })?,
+ _ => None,
+ };
+
+ let (all_scalar, string_array) = match &args[0] {
+ ColumnarValue::Array(a) => (false, Arc::clone(a)),
+ ColumnarValue::Scalar(s) => (true, s.to_array_of_size(1)?),
+ };
+
+ let result = match string_array.data_type() {
+ Utf8 => {
+ let arr = string_array.as_string::<i32>();
+ let builder =
+ StringBuilder::with_capacity(arr.len(), arr.get_buffer_memory_size());
+ string_to_array_scalar_args(&arr, delimiter, null_value, builder)
+ }
+ Utf8View => {
+ let arr = string_array.as_string_view();
+ let builder = StringViewBuilder::with_capacity(arr.len());
+ string_to_array_scalar_args(&arr, delimiter, null_value, builder)
+ }
+ LargeUtf8 => {
+ let arr = string_array.as_string::<i64>();
+ let builder = LargeStringBuilder::with_capacity(
+ arr.len(),
+ arr.get_buffer_memory_size(),
+ );
+ string_to_array_scalar_args(&arr, delimiter, null_value, builder)
+ }
other => {
exec_err!("unsupported type for string_to_array function as {other:?}")
}
+ }?;
+
+ if all_scalar {
+ ScalarValue::try_from_array(&result, 0).map(ColumnarValue::Scalar)
+ } else {
+ Ok(ColumnarValue::Array(result))
}
}
@@ -252,6 +316,201 @@ impl ScalarUDFImpl for StringToArray {
}
}
+/// Appends `value` to the string builder, or NULL if it matches `null_value`.
+#[inline(always)]
+fn append_part(
+ builder: &mut impl StringArrayBuilderType,
+ value: &str,
+ null_value: Option<&str>,
+) {
+ if null_value == Some(value) {
+ builder.append_null();
+ } else {
+ builder.append_value(value);
+ }
+}
+
+/// Optimized `string_to_array` implementation for the common case where
+/// delimiter and null_value are scalar values.
+fn string_to_array_scalar_args<'a, StringArrType, StringBuilderType>(
+ string_array: &StringArrType,
+ delimiter: Option<&str>,
+ null_value: Option<&str>,
+ string_builder: StringBuilderType,
+) -> Result<ArrayRef>
+where
+ StringArrType: StringArrayType<'a>,
+ StringBuilderType: StringArrayBuilderType,
+{
+ let mut list_builder = ListBuilder::new(string_builder);
+
+ match delimiter {
+ Some("") => {
+ // Empty delimiter: each non-empty string becomes a single-element list.
+ // Empty strings produce an empty array (PostgreSQL compat).
+ for i in 0..string_array.len() {
+ if string_array.is_null(i) {
+ list_builder.append(false);
+ continue;
+ }
+ let string = string_array.value(i);
+ if !string.is_empty() {
+ append_part(list_builder.values(), string, null_value);
+ }
+ list_builder.append(true);
+ }
+ }
+ Some(delimiter) => {
+ // Rather than using `str::split`, do the split ourselves using
+ // `memmem::Finder`. This allows pre-compiling the delimiter search
+ // pattern once and reusing it for all rows.
+ let finder = memchr::memmem::Finder::new(delimiter.as_bytes());
+ let delim_len = delimiter.len();
+
+ for i in 0..string_array.len() {
+ if string_array.is_null(i) {
+ list_builder.append(false);
+ continue;
+ }
+ let string = string_array.value(i);
+ if !string.is_empty() {
+ let bytes = string.as_bytes();
+ let mut start = 0;
+ for pos in finder.find_iter(bytes) {
+ append_part(
+ list_builder.values(),
+ &string[start..pos],
+ null_value,
+ );
+ start = pos + delim_len;
+ }
+ // Trailing part after last delimiter (or entire string if no
+ // delimiter was found).
+ append_part(list_builder.values(), &string[start..], null_value);
+ }
+ list_builder.append(true);
+ }
+ }
+ None => {
+ // NULL delimiter: split into individual characters.
+ for i in 0..string_array.len() {
+ if string_array.is_null(i) {
+ list_builder.append(false);
+ continue;
+ }
+ let string = string_array.value(i);
+ for (pos, c) in string.char_indices() {
+ append_part(
+ list_builder.values(),
+ &string[pos..pos + c.len_utf8()],
+ null_value,
+ );
+ }
+ list_builder.append(true);
+ }
+ }
+ }
+
+ Ok(Arc::new(list_builder.finish()) as ArrayRef)
+}
+
+/// Fallback path for `string_to_array` when delimiter and/or null_value
+/// are array columns rather than scalars.
+fn string_to_array_fallback(args: &[ArrayRef]) -> Result<ArrayRef> {
+ let null_value_array = args.get(2);
+
+ match args[0].data_type() {
+ Utf8 => {
+ let arr = args[0].as_string::<i32>();
+ let builder =
+ StringBuilder::with_capacity(arr.len(), arr.get_buffer_memory_size());
+ string_to_array_column_args(&arr, &args[1], null_value_array, builder)
+ }
+ Utf8View => {
+ let arr = args[0].as_string_view();
+ let builder = StringViewBuilder::with_capacity(arr.len());
+ string_to_array_column_args(&arr, &args[1], null_value_array, builder)
+ }
+ LargeUtf8 => {
+ let arr = args[0].as_string::<i64>();
+ let builder = LargeStringBuilder::with_capacity(
+ arr.len(),
+ arr.get_buffer_memory_size(),
+ );
+ string_to_array_column_args(&arr, &args[1], null_value_array, builder)
+ }
+ other => exec_err!("unsupported type for string_to_array function as {other:?}"),
+ }
+}
+
+fn string_to_array_column_args<'a, StringArrType, StringBuilderType>(
+ string_array: &StringArrType,
+ delimiter_array: &ArrayRef,
+ null_value_array: Option<&ArrayRef>,
+ string_builder: StringBuilderType,
+) -> Result<ArrayRef>
+where
+ StringArrType: StringArrayType<'a>,
+ StringBuilderType: StringArrayBuilderType,
+{
+ let mut list_builder = ListBuilder::new(string_builder);
+
+ for i in 0..string_array.len() {
+ if string_array.is_null(i) {
+ list_builder.append(false);
+ continue;
+ }
+
+ let string = string_array.value(i);
+ let delimiter = get_str_value(delimiter_array, i);
+ let null_value = null_value_array.and_then(|arr| get_str_value(arr, i));
+
+ match delimiter {
+ Some("") => {
+ if !string.is_empty() {
+ append_part(list_builder.values(), string, null_value);
+ }
+ }
+ Some(delimiter) => {
+ if !string.is_empty() {
+ for part in string.split(delimiter) {
+ append_part(list_builder.values(), part, null_value);
+ }
+ }
+ }
+ None => {
+ for (pos, c) in string.char_indices() {
+ append_part(
+ list_builder.values(),
+ &string[pos..pos + c.len_utf8()],
+ null_value,
+ );
+ }
+ }
+ }
+
+ list_builder.append(true);
+ }
+
+ Ok(Arc::new(list_builder.finish()) as ArrayRef)
+}
+
+/// Returns the string value at index `i` from a string array of any type.
+fn get_str_value(array: &ArrayRef, i: usize) -> Option<&str> {
+ if array.is_null(i) {
+ return None;
+ }
+ match array.data_type() {
+ Utf8 => Some(array.as_string::<i32>().value(i)),
+ LargeUtf8 => Some(array.as_string::<i64>().value(i)),
+ Utf8View => Some(array.as_string_view().value(i)),
+ other => {
+ debug_assert!(false, "unexpected type in get_str_value: {other:?}");
+ None
+ }
+ }
+}
+
fn array_to_string_inner(args: &[ArrayRef]) -> Result<ArrayRef> {
if args.len() < 2 || args.len() > 3 {
return exec_err!("array_to_string expects two or three arguments");
@@ -521,275 +780,6 @@ where
Ok(())
}
-/// String_to_array SQL function
-/// Splits string at occurrences of delimiter and returns an array of parts
-/// string_to_array('abc~@~def~@~ghi', '~@~') = '["abc", "def", "ghi"]'
-fn string_to_array_inner<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- if args.len() < 2 || args.len() > 3 {
- return exec_err!("string_to_array expects two or three arguments");
- }
-
- match args[0].data_type() {
- Utf8 => {
- let string_array = args[0].as_string::<T>();
- let builder = StringBuilder::with_capacity(
- string_array.len(),
- string_array.get_buffer_memory_size(),
- );
- string_to_array_inner_2::<&GenericStringArray<T>, StringBuilder>(
- args,
- &string_array,
- builder,
- )
- }
- Utf8View => {
- let string_array = args[0].as_string_view();
- let builder = StringViewBuilder::with_capacity(string_array.len());
- string_to_array_inner_2::<&StringViewArray, StringViewBuilder>(
- args,
- &string_array,
- builder,
- )
- }
- LargeUtf8 => {
- let string_array = args[0].as_string::<T>();
- let builder = LargeStringBuilder::with_capacity(
- string_array.len(),
- string_array.get_buffer_memory_size(),
- );
- string_to_array_inner_2::<&GenericStringArray<T>, LargeStringBuilder>(
- args,
- &string_array,
- builder,
- )
- }
- other => exec_err!(
- "unsupported type for first argument to string_to_array function as {other:?}"
- ),
- }
-}
-
-fn string_to_array_inner_2<'a, StringArrType, StringBuilderType>(
- args: &'a [ArrayRef],
- string_array: &StringArrType,
- string_builder: StringBuilderType,
-) -> Result<ArrayRef>
-where
- StringArrType: StringArrayType<'a>,
- StringBuilderType: StringArrayBuilderType,
-{
- match args[1].data_type() {
- Utf8 => {
- let delimiter_array = args[1].as_string::<i32>();
- if args.len() == 2 {
- string_to_array_impl::<
- StringArrType,
- &GenericStringArray<i32>,
- &StringViewArray,
- StringBuilderType,
- >(string_array, &delimiter_array, None, string_builder)
- } else {
- string_to_array_inner_3::<
- StringArrType,
- &GenericStringArray<i32>,
- StringBuilderType,
- >(args, string_array, &delimiter_array, string_builder)
- }
- }
- Utf8View => {
- let delimiter_array = args[1].as_string_view();
-
- if args.len() == 2 {
- string_to_array_impl::<
- StringArrType,
- &StringViewArray,
- &StringViewArray,
- StringBuilderType,
- >(string_array, &delimiter_array, None, string_builder)
- } else {
- string_to_array_inner_3::<
- StringArrType,
- &StringViewArray,
- StringBuilderType,
- >(args, string_array, &delimiter_array, string_builder)
- }
- }
- LargeUtf8 => {
- let delimiter_array = args[1].as_string::<i64>();
- if args.len() == 2 {
- string_to_array_impl::<
- StringArrType,
- &GenericStringArray<i64>,
- &StringViewArray,
- StringBuilderType,
- >(string_array, &delimiter_array, None, string_builder)
- } else {
- string_to_array_inner_3::<
- StringArrType,
- &GenericStringArray<i64>,
- StringBuilderType,
- >(args, string_array, &delimiter_array, string_builder)
- }
- }
- other => exec_err!(
- "unsupported type for second argument to string_to_array function as {other:?}"
- ),
- }
-}
-
-fn string_to_array_inner_3<'a, StringArrType, DelimiterArrType, StringBuilderType>(
- args: &'a [ArrayRef],
- string_array: &StringArrType,
- delimiter_array: &DelimiterArrType,
- string_builder: StringBuilderType,
-) -> Result<ArrayRef>
-where
- StringArrType: StringArrayType<'a>,
- DelimiterArrType: StringArrayType<'a>,
- StringBuilderType: StringArrayBuilderType,
-{
- match args[2].data_type() {
- Utf8 => {
- let null_type_array = Some(args[2].as_string::<i32>());
- string_to_array_impl::<
- StringArrType,
- DelimiterArrType,
- &GenericStringArray<i32>,
- StringBuilderType,
- >(
- string_array,
- delimiter_array,
- null_type_array,
- string_builder,
- )
- }
- Utf8View => {
- let null_type_array = Some(args[2].as_string_view());
- string_to_array_impl::<
- StringArrType,
- DelimiterArrType,
- &StringViewArray,
- StringBuilderType,
- >(
- string_array,
- delimiter_array,
- null_type_array,
- string_builder,
- )
- }
- LargeUtf8 => {
- let null_type_array = Some(args[2].as_string::<i64>());
- string_to_array_impl::<
- StringArrType,
- DelimiterArrType,
- &GenericStringArray<i64>,
- StringBuilderType,
- >(
- string_array,
- delimiter_array,
- null_type_array,
- string_builder,
- )
- }
- other => {
- exec_err!("unsupported type for string_to_array function as {other:?}")
- }
- }
-}
-
-fn string_to_array_impl<
- 'a,
- StringArrType,
- DelimiterArrType,
- NullValueArrType,
- StringBuilderType,
->(
- string_array: &StringArrType,
- delimiter_array: &DelimiterArrType,
- null_value_array: Option<NullValueArrType>,
- string_builder: StringBuilderType,
-) -> Result<ArrayRef>
-where
- StringArrType: StringArrayType<'a>,
- DelimiterArrType: StringArrayType<'a>,
- NullValueArrType: StringArrayType<'a>,
- StringBuilderType: StringArrayBuilderType,
-{
- let mut list_builder = ListBuilder::new(string_builder);
-
- match null_value_array {
- None => string_array.iter().zip(delimiter_array.iter()).for_each(
- |(string, delimiter)| match (string, delimiter) {
- (Some(string), Some("")) => {
- if !string.is_empty() {
- list_builder.values().append_value(string);
- }
- list_builder.append(true);
- }
- (Some(string), Some(delimiter)) => {
- if !string.is_empty() {
- string.split(delimiter).for_each(|s| {
- list_builder.values().append_value(s);
- });
- }
- list_builder.append(true);
- }
- (Some(string), None) => {
- string.chars().map(|c| c.to_string()).for_each(|c| {
- list_builder.values().append_value(c.as_str());
- });
- list_builder.append(true);
- }
- _ => list_builder.append(false),
- },
- ),
- Some(null_value_array) => string_array
- .iter()
- .zip(delimiter_array.iter())
- .zip(null_value_array.iter())
- .for_each(|((string, delimiter), null_value)| {
- match (string, delimiter) {
- (Some(string), Some("")) => {
- if !string.is_empty() {
- if Some(string) == null_value {
- list_builder.values().append_null();
- } else {
- list_builder.values().append_value(string);
- }
- }
- list_builder.append(true);
- }
- (Some(string), Some(delimiter)) => {
- if !string.is_empty() {
- string.split(delimiter).for_each(|s| {
- if Some(s) == null_value {
- list_builder.values().append_null();
- } else {
- list_builder.values().append_value(s);
- }
- });
- }
- list_builder.append(true);
- }
- (Some(string), None) => {
- string.chars().map(|c| c.to_string()).for_each(|c| {
- if Some(c.as_str()) == null_value {
- list_builder.values().append_null();
- } else {
- list_builder.values().append_value(c.as_str());
- }
- });
- list_builder.append(true);
- }
- _ => list_builder.append(false), // null value
- }
- }),
- };
-
- let list_array = list_builder.finish();
- Ok(Arc::new(list_array) as ArrayRef)
-}
-
trait StringArrayBuilderType: ArrayBuilder {
fn append_value(&mut self, val: &str);
diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt
index 1216e1e023..25136ca777 100644
--- a/datafusion/sqllogictest/test_files/array.slt
+++ b/datafusion/sqllogictest/test_files/array.slt
@@ -9358,6 +9358,74 @@ select string_to_list(e, 'm') from values;
[adipiscing]
NULL
+# string_to_array: single-char delimiter producing multiple elements
+query ?
+SELECT string_to_array('a,b,c', ',')
+----
+[a, b, c]
+
+# string_to_array: delimiter not found in input
+query ?
+SELECT string_to_array('abc', ',')
+----
+[abc]
+
+# string_to_array: empty string input
+query ?
+SELECT string_to_array('', ',')
+----
+[]
+
+# string_to_array: null_str matching multiple elements
+query ?
+SELECT string_to_array('a,NULL,b,NULL,c', ',', 'NULL')
+----
+[a, NULL, b, NULL, c]
+
+# string_to_array: null_str matching all elements
+query ?
+SELECT string_to_array('x,x,x', ',', 'x')
+----
+[NULL, NULL, NULL]
+
+# string_to_array: null_str with empty-string delimiter
+query ?
+SELECT string_to_array('abc', '', 'abc')
+----
+[NULL]
+
+# string_to_array: NULL string input
+query ?
+SELECT string_to_array(NULL, ',')
+----
+NULL
+
+# string_to_array: columnar delimiter
+query ??
+SELECT string_to_array('a,b,c', col1), string_to_array('a::b::c', col2)
+ FROM (VALUES (',', '::')) AS t(col1, col2)
+----
+[a, b, c] [a, b, c]
+
+# string_to_array: columnar null_str
+query ?
+SELECT string_to_array('a,NULL,b', ',', col1)
+ FROM (VALUES ('NULL')) AS t(col1)
+----
+[a, NULL, b]
+
+# string_to_array: adjacent delimiters produce empty strings
+query ?
+SELECT string_to_array('a,,b', ',')
+----
+[a, , b]
+
+# string_to_array: delimiter at start and end
+query ?
+SELECT string_to_array(',a,b,', ',')
+----
+[, a, b, ]
+
# array_resize scalar function #1
query ?
select array_resize(make_array(1, 2, 3), 1);