Skip to content

Apache DataFusion SQL Query Engine

perf: Optimize `split_part` for `Utf8View` (#21420)

Neil Conway190b1049a637
diff --git a/datafusion/functions/benches/split_part.rs b/datafusion/functions/benches/split_part.rs
index 0f4998effc..d578339368 100644
--- a/datafusion/functions/benches/split_part.rs
+++ b/datafusion/functions/benches/split_part.rs
@@ -210,6 +210,23 @@ fn criterion_benchmark(c: &mut Criterion) {
);
}
+ // Utf8View, very long parts (256 bytes), position 1
+ {
+ let strings = gen_string_array(N_ROWS, 5, 256, ".", true);
+ let delimiter = ColumnarValue::Scalar(ScalarValue::Utf8View(Some(".".into())));
+ let position = ColumnarValue::Scalar(ScalarValue::Int64(Some(1)));
+ bench_split_part(
+ &mut group,
+ &split_part_func,
+ &config_options,
+ "scalar_utf8view_very_long_parts",
+ "pos_first",
+ strings,
+ delimiter,
+ position,
+ );
+ }
+
// ── Array delimiter and position ─────────────────
// Utf8, single-char delimiter, array args
diff --git a/datafusion/functions/src/string/split_part.rs b/datafusion/functions/src/string/split_part.rs
index 972a10c264..1994c65bcf 100644
--- a/datafusion/functions/src/string/split_part.rs
+++ b/datafusion/functions/src/string/split_part.rs
@@ -17,9 +17,11 @@
use crate::utils::utf8_to_str_type;
use arrow::array::{
- Array, ArrayRef, AsArray, GenericStringBuilder, Int64Array, StringArrayType,
- StringLikeArrayBuilder, StringViewBuilder, new_null_array,
+ Array, ArrayRef, AsArray, ByteView, GenericStringBuilder, Int64Array,
+ StringArrayType, StringLikeArrayBuilder, StringViewArray, StringViewBuilder,
+ make_view, new_null_array,
};
+use arrow::buffer::ScalarBuffer;
use arrow::datatypes::DataType;
use datafusion_common::ScalarValue;
use datafusion_common::cast::as_int64_array;
@@ -279,12 +281,9 @@ fn split_part_scalar(
}
let result = match string_array.data_type() {
- DataType::Utf8View => split_part_scalar_impl(
- string_array.as_string_view(),
- delimiter,
- position,
- StringViewBuilder::with_capacity(string_array.len()),
- ),
+ DataType::Utf8View => {
+ split_part_scalar_view(string_array.as_string_view(), delimiter, position)
+ }
DataType::Utf8 => {
let arr = string_array.as_string::<i32>();
// Conservative under-estimate for data capacity: split_part output
@@ -425,6 +424,116 @@ fn rsplit_nth_finder<'a>(
}
}
+/// Zero-copy scalar fast path for `StringViewArray` inputs.
+///
+/// Instead of copying substring bytes into a new buffer, constructs
+/// `StringView` entries that point back into the original array's data
+/// buffers.
+fn split_part_scalar_view(
+ string_view_array: &StringViewArray,
+ delimiter: &str,
+ position: i64,
+) -> Result<ArrayRef> {
+ let len = string_view_array.len();
+ let mut views_buf = Vec::with_capacity(len);
+ let views = string_view_array.views();
+
+ if delimiter.is_empty() {
+ // PostgreSQL: empty delimiter treats input as a single field.
+ let empty_view = make_view(b"", 0, 0);
+ let return_input = position == 1 || position == -1;
+ for i in 0..len {
+ if string_view_array.is_null(i) {
+ views_buf.push(0);
+ } else if return_input {
+ views_buf.push(views[i]);
+ } else {
+ views_buf.push(empty_view);
+ }
+ }
+ } else if position > 0 {
+ let idx: usize = (position - 1).try_into().map_err(|_| {
+ exec_datafusion_err!(
+ "split_part index {position} exceeds maximum supported value"
+ )
+ })?;
+ let finder = memmem::Finder::new(delimiter.as_bytes());
+ split_view_loop(string_view_array, views, &mut views_buf, |s| {
+ split_nth_finder(s, &finder, delimiter.len(), idx)
+ });
+ } else {
+ let idx: usize = (position.unsigned_abs() - 1).try_into().map_err(|_| {
+ exec_datafusion_err!(
+ "split_part index {position} exceeds minimum supported value"
+ )
+ })?;
+ let finder_rev = memmem::FinderRev::new(delimiter.as_bytes());
+ split_view_loop(string_view_array, views, &mut views_buf, |s| {
+ rsplit_nth_finder(s, &finder_rev, delimiter.len(), idx)
+ });
+ }
+
+ let views_buf = ScalarBuffer::from(views_buf);
+
+ // Nulls pass through unchanged, so we can use the input's null array.
+ let nulls = string_view_array.nulls().cloned();
+
+ // Safety: each view is either copied unchanged from the input, or built
+ // by `substr_view` from a substring that is a contiguous sub-range of the
+ // original string value stored in the input's data buffers.
+ unsafe {
+ Ok(Arc::new(StringViewArray::new_unchecked(
+ views_buf,
+ string_view_array.data_buffers().to_vec(),
+ nulls,
+ )) as ArrayRef)
+ }
+}
+
+/// Creates a `StringView` referencing a substring of an existing view's buffer.
+/// For substrings ≤ 12 bytes, creates an inline view instead.
+#[inline]
+fn substr_view(original_view: &u128, substr: &str, start_offset: u32) -> u128 {
+ if substr.len() > 12 {
+ let view = ByteView::from(*original_view);
+ make_view(
+ substr.as_bytes(),
+ view.buffer_index,
+ view.offset + start_offset,
+ )
+ } else {
+ make_view(substr.as_bytes(), 0, 0)
+ }
+}
+
+/// Applies `split_fn` to each non-null string and appends the resulting view to
+/// `views_buf`.
+#[inline(always)]
+fn split_view_loop<F>(
+ string_view_array: &StringViewArray,
+ views: &[u128],
+ views_buf: &mut Vec<u128>,
+ split_fn: F,
+) where
+ F: Fn(&str) -> Option<&str>,
+{
+ let empty_view = make_view(b"", 0, 0);
+ for (i, raw_view) in views.iter().enumerate() {
+ if string_view_array.is_null(i) {
+ views_buf.push(0);
+ continue;
+ }
+ let string = string_view_array.value(i);
+ match split_fn(string) {
+ Some(substr) => {
+ let start_offset = substr.as_ptr() as usize - string.as_ptr() as usize;
+ views_buf.push(substr_view(raw_view, substr, start_offset as u32));
+ }
+ None => views_buf.push(empty_view),
+ }
+ }
+}
+
fn split_part_impl<'a, StringArrType, DelimiterArrType, B>(
string_array: &StringArrType,
delimiter_array: &DelimiterArrType,
@@ -490,7 +599,7 @@ where
#[cfg(test)]
mod tests {
- use arrow::array::{Array, StringArray};
+ use arrow::array::{Array, AsArray, StringArray, StringViewArray};
use arrow::datatypes::DataType::Utf8;
use datafusion_common::ScalarValue;
@@ -686,4 +795,31 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_split_part_stringview_sliced() -> Result<()> {
+ use super::split_part_scalar_view;
+
+ let strings: StringViewArray = vec![
+ Some("skip_this.value"),
+ Some("this_is_a_long_prefix.suffix"),
+ Some("short.val"),
+ Some("another_long_result.rest"),
+ None,
+ ]
+ .into_iter()
+ .collect();
+
+ // Slice off the first element to get a non-zero offset array.
+ let sliced = strings.slice(1, 4);
+ let result = split_part_scalar_view(&sliced, ".", 1)?;
+ let result = result.as_string_view();
+ assert_eq!(result.len(), 4);
+ assert_eq!(result.value(0), "this_is_a_long_prefix");
+ assert_eq!(result.value(1), "short");
+ assert_eq!(result.value(2), "another_long_result");
+ assert!(result.is_null(3));
+
+ Ok(())
+ }
}
diff --git a/datafusion/sqllogictest/test_files/string/string_view.slt b/datafusion/sqllogictest/test_files/string/string_view.slt
index 086f37d6c3..126c4bcafb 100644
--- a/datafusion/sqllogictest/test_files/string/string_view.slt
+++ b/datafusion/sqllogictest/test_files/string/string_view.slt
@@ -954,6 +954,97 @@ SELECT arrow_typeof(split_part(arrow_cast('a.b.c', 'Utf8View'), '.', 2));
----
Utf8View
+# SPLIT_PART with Utf8View column (exercises the array fast path)
+query T
+SELECT split_part(column1_utf8view, 'ph', 1) FROM test;
+----
+Andrew
+Xiangpeng
+Ra
+(empty)
+NULL
+
+query T
+SELECT split_part(column1_utf8view, 'ph', 2) FROM test;
+----
+(empty)
+(empty)
+ael
+(empty)
+NULL
+
+# Negative position
+query T
+SELECT split_part(column1_utf8view, 'ph', -1) FROM test;
+----
+Andrew
+Xiangpeng
+ael
+(empty)
+NULL
+
+# Delimiter not found returns full string
+query T
+SELECT split_part(column1_utf8view, 'ZZZ', 1) FROM test;
+----
+Andrew
+Xiangpeng
+Raphael
+(empty)
+NULL
+
+# Empty delimiter with column
+query T
+SELECT split_part(column1_utf8view, '', 1) FROM test;
+----
+Andrew
+Xiangpeng
+Raphael
+(empty)
+NULL
+
+# Single-char delimiter with column
+query T
+SELECT split_part(column1_utf8view, 'a', 1) FROM test;
+----
+Andrew
+Xi
+R
+(empty)
+NULL
+
+# Verify array path also returns Utf8View
+query T
+SELECT arrow_typeof(split_part(column1_utf8view, '.', 1)) FROM test LIMIT 1;
+----
+Utf8View
+
+# Long strings (>12 bytes) exercise out-of-line StringView construction in split_part
+query T
+SELECT split_part(arrow_cast(column1, 'Utf8View'), '.', 1) FROM
+ (VALUES ('this_is_over_12.suffix'), ('short.val'), (NULL)) AS t(column1);
+----
+this_is_over_12
+short
+NULL
+
+query T
+SELECT split_part(arrow_cast(column1, 'Utf8View'), '.', -1) FROM
+ (VALUES ('prefix.this_is_over_12'), ('a.short'), (NULL)) AS t(column1);
+----
+this_is_over_12
+short
+NULL
+
+# Results at the 12-byte inline/out-of-line boundary
+query T
+SELECT split_part(arrow_cast(column1, 'Utf8View'), '.', 1) FROM
+ (VALUES ('exactly12byt.rest'), ('thirteen_byte.rest'), ('twelve_bytes.rest')) AS t(column1);
+----
+exactly12byt
+thirteen_byte
+twelve_bytes
+
## Ensure no casts for STRPOS
query TT
EXPLAIN SELECT