Skip to content

Apache DataFusion SQL Query Engine

perf: Use bulk-NULL builder in `replace` (#21849)

Neil Conway720aaff915b6
diff --git a/datafusion/functions/benches/replace.rs b/datafusion/functions/benches/replace.rs
index 55fbd6ae57..7ad198995a 100644
--- a/datafusion/functions/benches/replace.rs
+++ b/datafusion/functions/benches/replace.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::OffsetSizeTrait;
+use arrow::array::{GenericStringArray, OffsetSizeTrait, StringViewArray};
use arrow::datatypes::{DataType, Field};
use arrow::util::bench_util::{
create_string_array_with_len, create_string_view_array_with_len,
@@ -29,31 +29,58 @@ use std::hint::black_box;
use std::sync::Arc;
use std::time::Duration;
+/// Build a string array, dropping the null buffer when `null_density == 0.0`
+fn make_string_array<O: OffsetSizeTrait>(
+ size: usize,
+ null_density: f32,
+ str_len: usize,
+) -> GenericStringArray<O> {
+ let arr = create_string_array_with_len::<O>(size, null_density, str_len);
+ if null_density == 0.0 {
+ let (offsets, values, _) = arr.into_parts();
+ GenericStringArray::<O>::new(offsets, values, None)
+ } else {
+ arr
+ }
+}
+
+fn make_string_view_array(
+ size: usize,
+ null_density: f32,
+ str_len: usize,
+) -> StringViewArray {
+ let arr = create_string_view_array_with_len(size, null_density, str_len, false);
+ if null_density == 0.0 {
+ let (views, buffers, _) = arr.into_parts();
+ StringViewArray::new(views, buffers, None)
+ } else {
+ arr
+ }
+}
+
fn create_args<O: OffsetSizeTrait>(
size: usize,
str_len: usize,
force_view_types: bool,
from_len: usize,
to_len: usize,
+ null_density: f32,
) -> Vec<ColumnarValue> {
+ // Apply `null_density` only to the string column; `from` and `to` are
+ // typically not NULL in real-world workloads.
if force_view_types {
- let string_array =
- Arc::new(create_string_view_array_with_len(size, 0.1, str_len, false));
- let from_array = Arc::new(create_string_view_array_with_len(
- size, 0.1, from_len, false,
- ));
- let to_array =
- Arc::new(create_string_view_array_with_len(size, 0.1, to_len, false));
+ let string_array = Arc::new(make_string_view_array(size, null_density, str_len));
+ let from_array = Arc::new(make_string_view_array(size, 0.0, from_len));
+ let to_array = Arc::new(make_string_view_array(size, 0.0, to_len));
vec![
ColumnarValue::Array(string_array),
ColumnarValue::Array(from_array),
ColumnarValue::Array(to_array),
]
} else {
- let string_array =
- Arc::new(create_string_array_with_len::<O>(size, 0.1, str_len));
- let from_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, from_len));
- let to_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, to_len));
+ let string_array = Arc::new(make_string_array::<O>(size, null_density, str_len));
+ let from_array = Arc::new(make_string_array::<O>(size, 0.0, from_len));
+ let to_array = Arc::new(make_string_array::<O>(size, 0.0, to_len));
vec![
ColumnarValue::Array(string_array),
@@ -90,98 +117,50 @@ fn criterion_benchmark(c: &mut Criterion) {
group.sample_size(10);
group.measurement_time(Duration::from_secs(10));
- // ASCII single character replacement (fast path)
- let str_len = 32;
- let args = create_args::<i32>(size, str_len, false, 1, 1);
- group.bench_function(
- format!("replace_string_ascii_single [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- // Multi-character strings (general path)
- let args = create_args::<i32>(size, str_len, true, 3, 5);
- group.bench_function(
- format!("replace_string_view [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- let args = create_args::<i32>(size, str_len, false, 3, 5);
- group.bench_function(
- format!("replace_string [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- let args = create_args::<i64>(size, str_len, false, 3, 5);
- group.bench_function(
- format!("replace_large_string [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- // Larger strings
- let str_len = 128;
- let args = create_args::<i32>(size, str_len, false, 1, 1);
- group.bench_function(
- format!("replace_string_ascii_single [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- let args = create_args::<i32>(size, str_len, true, 3, 5);
- group.bench_function(
- format!("replace_string_view [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- let args = create_args::<i32>(size, str_len, false, 3, 5);
- group.bench_function(
- format!("replace_string [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
-
- let args = create_args::<i64>(size, str_len, false, 3, 5);
- group.bench_function(
- format!("replace_large_string [size={size}, str_len={str_len}]"),
- |b| {
- b.iter(|| {
- let args_cloned = args.clone();
- black_box(invoke_replace_with_args(args_cloned, size))
- })
- },
- );
+ for &nulls in &[0.0_f32, 0.2] {
+ for &str_len in &[32_usize, 128] {
+ // ASCII single character replacement (fast path)
+ let args = create_args::<i32>(size, str_len, false, 1, 1, nulls);
+ group.bench_function(
+ format!(
+ "replace_string_ascii_single [size={size}, str_len={str_len}, nulls={nulls}]"
+ ),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(invoke_replace_with_args(args_cloned, size))
+ })
+ },
+ );
+
+ // Multi-character strings (general path)
+ let args = create_args::<i32>(size, str_len, true, 3, 5, nulls);
+ group.bench_function(
+ format!(
+ "replace_string_view [size={size}, str_len={str_len}, nulls={nulls}]"
+ ),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(invoke_replace_with_args(args_cloned, size))
+ })
+ },
+ );
+
+ let args = create_args::<i32>(size, str_len, false, 3, 5, nulls);
+ group.bench_function(
+ format!(
+ "replace_string [size={size}, str_len={str_len}, nulls={nulls}]"
+ ),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(invoke_replace_with_args(args_cloned, size))
+ })
+ },
+ );
+ }
+ }
group.finish();
}
diff --git a/datafusion/functions/src/string/replace.rs b/datafusion/functions/src/string/replace.rs
index cca68844a1..f588b8c3cc 100644
--- a/datafusion/functions/src/string/replace.rs
+++ b/datafusion/functions/src/string/replace.rs
@@ -17,9 +17,11 @@
use std::sync::Arc;
-use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};
+use arrow::array::{Array, ArrayRef, OffsetSizeTrait};
+use arrow::buffer::NullBuffer;
use arrow::datatypes::DataType;
+use crate::strings::GenericStringArrayBuilder;
use crate::utils::{make_scalar_function, utf8_to_str_type};
use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
use datafusion_common::types::logical_string;
@@ -160,25 +162,44 @@ fn replace_view(args: &[ArrayRef]) -> Result<ArrayRef> {
let from_array = as_string_view_array(&args[1])?;
let to_array = as_string_view_array(&args[2])?;
- let mut builder = GenericStringBuilder::<i32>::new();
+ let len = string_array.len();
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(len, 0);
let mut buffer = String::new();
+ let nulls = NullBuffer::union(
+ NullBuffer::union(string_array.nulls(), from_array.nulls()).as_ref(),
+ to_array.nulls(),
+ );
- for ((string, from), to) in string_array
- .iter()
- .zip(from_array.iter())
- .zip(to_array.iter())
- {
- match (string, from, to) {
- (Some(string), Some(from), Some(to)) => {
- buffer.clear();
- replace_into_string(&mut buffer, string, from, to);
- builder.append_value(&buffer);
+ // Hoist the nulls.is_some() check out of the loop. LLVM does not always
+ // unswitch this loop on its own (the Utf8View body is large enough to
+ // exceed its cost-benefit threshold).
+ if let Some(nulls_ref) = nulls.as_ref() {
+ for i in 0..len {
+ if nulls_ref.is_null(i) {
+ builder.append_placeholder();
+ continue;
}
- _ => builder.append_null(),
+ // SAFETY: union of input nulls is non-null at i, so each input is too.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ buffer.clear();
+ replace_into_string(&mut buffer, string, from, to);
+ builder.append_value(&buffer);
+ }
+ } else {
+ for i in 0..len {
+ // SAFETY: i < len, and no input has a null buffer.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ buffer.clear();
+ replace_into_string(&mut buffer, string, from, to);
+ builder.append_value(&buffer);
}
}
- Ok(Arc::new(builder.finish()) as ArrayRef)
+ Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)
}
/// Replaces all occurrences in string of substring from with substring to.
@@ -188,25 +209,44 @@ fn replace<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
let from_array = as_generic_string_array::<T>(&args[1])?;
let to_array = as_generic_string_array::<T>(&args[2])?;
- let mut builder = GenericStringBuilder::<T>::new();
+ let len = string_array.len();
+ let mut builder = GenericStringArrayBuilder::<T>::with_capacity(len, 0);
let mut buffer = String::new();
+ let nulls = NullBuffer::union(
+ NullBuffer::union(string_array.nulls(), from_array.nulls()).as_ref(),
+ to_array.nulls(),
+ );
- for ((string, from), to) in string_array
- .iter()
- .zip(from_array.iter())
- .zip(to_array.iter())
- {
- match (string, from, to) {
- (Some(string), Some(from), Some(to)) => {
- buffer.clear();
- replace_into_string(&mut buffer, string, from, to);
- builder.append_value(&buffer);
+ // Hoist the nulls.is_some() check out of the loop. LLVM unswitches this
+ // automatically today, but kept explicit so the no-nulls fast path is not
+ // contingent on the optimizer's cost heuristic.
+ if let Some(nulls_ref) = nulls.as_ref() {
+ for i in 0..len {
+ if nulls_ref.is_null(i) {
+ builder.append_placeholder();
+ continue;
}
- _ => builder.append_null(),
+ // SAFETY: union of input nulls is non-null at i, so each input is too.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ buffer.clear();
+ replace_into_string(&mut buffer, string, from, to);
+ builder.append_value(&buffer);
+ }
+ } else {
+ for i in 0..len {
+ // SAFETY: i < len, and no input has a null buffer.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ buffer.clear();
+ replace_into_string(&mut buffer, string, from, to);
+ builder.append_value(&buffer);
}
}
- Ok(Arc::new(builder.finish()) as ArrayRef)
+ Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)
}
/// Helper function to perform string replacement into a reusable String buffer