Apache DataFusion SQL Query Engine
perf: Use bulk-NULL builder in `replace` (#21849)
Neil Conway720aaff915b6
diff --git a/datafusion/functions/benches/replace.rs b/datafusion/functions/benches/replace.rsindex 55fbd6ae57..7ad198995a 100644--- a/datafusion/functions/benches/replace.rs+++ b/datafusion/functions/benches/replace.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-use arrow::array::OffsetSizeTrait;+use arrow::array::{GenericStringArray, OffsetSizeTrait, StringViewArray};use arrow::datatypes::{DataType, Field};use arrow::util::bench_util::{create_string_array_with_len, create_string_view_array_with_len,@@ -29,31 +29,58 @@ use std::hint::black_box;use std::sync::Arc;use std::time::Duration;+/// Build a string array, dropping the null buffer when `null_density == 0.0`+fn make_string_array<O: OffsetSizeTrait>(+ size: usize,+ null_density: f32,+ str_len: usize,+) -> GenericStringArray<O> {+ let arr = create_string_array_with_len::<O>(size, null_density, str_len);+ if null_density == 0.0 {+ let (offsets, values, _) = arr.into_parts();+ GenericStringArray::<O>::new(offsets, values, None)+ } else {+ arr+ }+}++fn make_string_view_array(+ size: usize,+ null_density: f32,+ str_len: usize,+) -> StringViewArray {+ let arr = create_string_view_array_with_len(size, null_density, str_len, false);+ if null_density == 0.0 {+ let (views, buffers, _) = arr.into_parts();+ StringViewArray::new(views, buffers, None)+ } else {+ arr+ }+}+fn create_args<O: OffsetSizeTrait>(size: usize,str_len: usize,force_view_types: bool,from_len: usize,to_len: usize,+ null_density: f32,) -> Vec<ColumnarValue> {+ // Apply `null_density` only to the string column; `from` and `to` are+ // typically not NULL in real-world workloads.if force_view_types {- let string_array =- Arc::new(create_string_view_array_with_len(size, 0.1, str_len, false));- let from_array = Arc::new(create_string_view_array_with_len(- size, 0.1, from_len, false,- ));- let to_array =- Arc::new(create_string_view_array_with_len(size, 0.1, to_len, false));+ let string_array = Arc::new(make_string_view_array(size, null_density, str_len));+ let from_array = Arc::new(make_string_view_array(size, 0.0, from_len));+ let to_array = Arc::new(make_string_view_array(size, 0.0, to_len));vec![ColumnarValue::Array(string_array),ColumnarValue::Array(from_array),ColumnarValue::Array(to_array),]} else {- let string_array =- Arc::new(create_string_array_with_len::<O>(size, 0.1, str_len));- let from_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, from_len));- let to_array = Arc::new(create_string_array_with_len::<O>(size, 0.1, to_len));+ let string_array = Arc::new(make_string_array::<O>(size, null_density, str_len));+ let from_array = Arc::new(make_string_array::<O>(size, 0.0, from_len));+ let to_array = Arc::new(make_string_array::<O>(size, 0.0, to_len));vec![ColumnarValue::Array(string_array),@@ -90,98 +117,50 @@ fn criterion_benchmark(c: &mut Criterion) {group.sample_size(10);group.measurement_time(Duration::from_secs(10));- // ASCII single character replacement (fast path)- let str_len = 32;- let args = create_args::<i32>(size, str_len, false, 1, 1);- group.bench_function(- format!("replace_string_ascii_single [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- // Multi-character strings (general path)- let args = create_args::<i32>(size, str_len, true, 3, 5);- group.bench_function(- format!("replace_string_view [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- let args = create_args::<i32>(size, str_len, false, 3, 5);- group.bench_function(- format!("replace_string [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- let args = create_args::<i64>(size, str_len, false, 3, 5);- group.bench_function(- format!("replace_large_string [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- // Larger strings- let str_len = 128;- let args = create_args::<i32>(size, str_len, false, 1, 1);- group.bench_function(- format!("replace_string_ascii_single [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- let args = create_args::<i32>(size, str_len, true, 3, 5);- group.bench_function(- format!("replace_string_view [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- let args = create_args::<i32>(size, str_len, false, 3, 5);- group.bench_function(- format!("replace_string [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );-- let args = create_args::<i64>(size, str_len, false, 3, 5);- group.bench_function(- format!("replace_large_string [size={size}, str_len={str_len}]"),- |b| {- b.iter(|| {- let args_cloned = args.clone();- black_box(invoke_replace_with_args(args_cloned, size))- })- },- );+ for &nulls in &[0.0_f32, 0.2] {+ for &str_len in &[32_usize, 128] {+ // ASCII single character replacement (fast path)+ let args = create_args::<i32>(size, str_len, false, 1, 1, nulls);+ group.bench_function(+ format!(+ "replace_string_ascii_single [size={size}, str_len={str_len}, nulls={nulls}]"+ ),+ |b| {+ b.iter(|| {+ let args_cloned = args.clone();+ black_box(invoke_replace_with_args(args_cloned, size))+ })+ },+ );++ // Multi-character strings (general path)+ let args = create_args::<i32>(size, str_len, true, 3, 5, nulls);+ group.bench_function(+ format!(+ "replace_string_view [size={size}, str_len={str_len}, nulls={nulls}]"+ ),+ |b| {+ b.iter(|| {+ let args_cloned = args.clone();+ black_box(invoke_replace_with_args(args_cloned, size))+ })+ },+ );++ let args = create_args::<i32>(size, str_len, false, 3, 5, nulls);+ group.bench_function(+ format!(+ "replace_string [size={size}, str_len={str_len}, nulls={nulls}]"+ ),+ |b| {+ b.iter(|| {+ let args_cloned = args.clone();+ black_box(invoke_replace_with_args(args_cloned, size))+ })+ },+ );+ }+ }group.finish();}diff --git a/datafusion/functions/src/string/replace.rs b/datafusion/functions/src/string/replace.rsindex cca68844a1..f588b8c3cc 100644--- a/datafusion/functions/src/string/replace.rs+++ b/datafusion/functions/src/string/replace.rs@@ -17,9 +17,11 @@use std::sync::Arc;-use arrow::array::{ArrayRef, GenericStringBuilder, OffsetSizeTrait};+use arrow::array::{Array, ArrayRef, OffsetSizeTrait};+use arrow::buffer::NullBuffer;use arrow::datatypes::DataType;+use crate::strings::GenericStringArrayBuilder;use crate::utils::{make_scalar_function, utf8_to_str_type};use datafusion_common::cast::{as_generic_string_array, as_string_view_array};use datafusion_common::types::logical_string;@@ -160,25 +162,44 @@ fn replace_view(args: &[ArrayRef]) -> Result<ArrayRef> {let from_array = as_string_view_array(&args[1])?;let to_array = as_string_view_array(&args[2])?;- let mut builder = GenericStringBuilder::<i32>::new();+ let len = string_array.len();+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(len, 0);let mut buffer = String::new();+ let nulls = NullBuffer::union(+ NullBuffer::union(string_array.nulls(), from_array.nulls()).as_ref(),+ to_array.nulls(),+ );- for ((string, from), to) in string_array- .iter()- .zip(from_array.iter())- .zip(to_array.iter())- {- match (string, from, to) {- (Some(string), Some(from), Some(to)) => {- buffer.clear();- replace_into_string(&mut buffer, string, from, to);- builder.append_value(&buffer);+ // Hoist the nulls.is_some() check out of the loop. LLVM does not always+ // unswitch this loop on its own (the Utf8View body is large enough to+ // exceed its cost-benefit threshold).+ if let Some(nulls_ref) = nulls.as_ref() {+ for i in 0..len {+ if nulls_ref.is_null(i) {+ builder.append_placeholder();+ continue;}- _ => builder.append_null(),+ // SAFETY: union of input nulls is non-null at i, so each input is too.+ let string = unsafe { string_array.value_unchecked(i) };+ let from = unsafe { from_array.value_unchecked(i) };+ let to = unsafe { to_array.value_unchecked(i) };+ buffer.clear();+ replace_into_string(&mut buffer, string, from, to);+ builder.append_value(&buffer);+ }+ } else {+ for i in 0..len {+ // SAFETY: i < len, and no input has a null buffer.+ let string = unsafe { string_array.value_unchecked(i) };+ let from = unsafe { from_array.value_unchecked(i) };+ let to = unsafe { to_array.value_unchecked(i) };+ buffer.clear();+ replace_into_string(&mut buffer, string, from, to);+ builder.append_value(&buffer);}}- Ok(Arc::new(builder.finish()) as ArrayRef)+ Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)}/// Replaces all occurrences in string of substring from with substring to.@@ -188,25 +209,44 @@ fn replace<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {let from_array = as_generic_string_array::<T>(&args[1])?;let to_array = as_generic_string_array::<T>(&args[2])?;- let mut builder = GenericStringBuilder::<T>::new();+ let len = string_array.len();+ let mut builder = GenericStringArrayBuilder::<T>::with_capacity(len, 0);let mut buffer = String::new();+ let nulls = NullBuffer::union(+ NullBuffer::union(string_array.nulls(), from_array.nulls()).as_ref(),+ to_array.nulls(),+ );- for ((string, from), to) in string_array- .iter()- .zip(from_array.iter())- .zip(to_array.iter())- {- match (string, from, to) {- (Some(string), Some(from), Some(to)) => {- buffer.clear();- replace_into_string(&mut buffer, string, from, to);- builder.append_value(&buffer);+ // Hoist the nulls.is_some() check out of the loop. LLVM unswitches this+ // automatically today, but kept explicit so the no-nulls fast path is not+ // contingent on the optimizer's cost heuristic.+ if let Some(nulls_ref) = nulls.as_ref() {+ for i in 0..len {+ if nulls_ref.is_null(i) {+ builder.append_placeholder();+ continue;}- _ => builder.append_null(),+ // SAFETY: union of input nulls is non-null at i, so each input is too.+ let string = unsafe { string_array.value_unchecked(i) };+ let from = unsafe { from_array.value_unchecked(i) };+ let to = unsafe { to_array.value_unchecked(i) };+ buffer.clear();+ replace_into_string(&mut buffer, string, from, to);+ builder.append_value(&buffer);+ }+ } else {+ for i in 0..len {+ // SAFETY: i < len, and no input has a null buffer.+ let string = unsafe { string_array.value_unchecked(i) };+ let from = unsafe { from_array.value_unchecked(i) };+ let to = unsafe { to_array.value_unchecked(i) };+ buffer.clear();+ replace_into_string(&mut buffer, string, from, to);+ builder.append_value(&buffer);}}- Ok(Arc::new(builder.finish()) as ArrayRef)+ Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)}/// Helper function to perform string replacement into a reusable String buffer