Apache DataFusion SQL Query Engine
perf: Optimize NULL handling in `substr` (#21519)
Neil Conway89397269299b
diff --git a/datafusion/functions/src/string/common.rs b/datafusion/functions/src/string/common.rsindex cc97c1b2c1..bda6b429ac 100644--- a/datafusion/functions/src/string/common.rs+++ b/datafusion/functions/src/string/common.rs@@ -19,7 +19,7 @@use std::sync::Arc;-use crate::strings::make_and_append_view;+use crate::strings::append_view;use arrow::array::{Array, ArrayRef, GenericStringArray, GenericStringBuilder, NullBufferBuilder,OffsetSizeTrait, StringViewArray, StringViewBuilder, new_null_array,@@ -152,13 +152,8 @@ fn string_view_trim<Tr: Trimmer>(args: &[ArrayRef]) -> Result<ArrayRef> {{if let Some(src_str) = src_str_opt {let (trimmed, offset) = Tr::trim_ascii_char(src_str, b' ');- make_and_append_view(- &mut views_buf,- &mut null_builder,- raw_view,- trimmed,- offset,- );+ append_view(&mut views_buf, raw_view, trimmed, offset);+ null_builder.append_non_null();} else {null_builder.append_null();views_buf.push(0);@@ -204,13 +199,8 @@ fn string_view_trim<Tr: Trimmer>(args: &[ArrayRef]) -> Result<ArrayRef> {pattern.clear();pattern.extend(characters.chars());let (trimmed, offset) = Tr::trim(src_str, &pattern);- make_and_append_view(- &mut views_buf,- &mut null_builder,- raw_view,- trimmed,- offset,- );+ append_view(&mut views_buf, raw_view, trimmed, offset);+ null_builder.append_non_null();} else {null_builder.append_null();views_buf.push(0);@@ -261,7 +251,8 @@ fn trim_and_append_view<Tr: Trimmer>() {if let Some(src_str) = src_str_opt {let (trimmed, offset) = Tr::trim(src_str, pattern);- make_and_append_view(views_buf, null_builder, original_view, trimmed, offset);+ append_view(views_buf, original_view, trimmed, offset);+ null_builder.append_non_null();} else {null_builder.append_null();views_buf.push(0);diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rsindex b0e988a3a8..ebc58490b2 100644--- a/datafusion/functions/src/strings.rs+++ b/datafusion/functions/src/strings.rs@@ -21,7 +21,7 @@ use datafusion_common::{Result, exec_datafusion_err, internal_err};use arrow::array::{Array, ArrayAccessor, ArrayDataBuilder, BinaryArray, ByteView, LargeStringArray,- NullBufferBuilder, StringArray, StringViewArray, StringViewBuilder, make_view,+ StringArray, StringViewArray, StringViewBuilder, make_view,};use arrow::buffer::{MutableBuffer, NullBuffer};use arrow::datatypes::DataType;@@ -372,7 +372,9 @@ impl LargeStringArrayBuilder {}}-/// Append a new view to the views buffer with the given substr+/// Append a new view to the views buffer with the given substr.+///+/// Callers are responsible for their own null tracking.////// # Safety///@@ -381,13 +383,15 @@ impl LargeStringArrayBuilder {////// # Arguments/// - views_buffer: The buffer to append the new view to-/// - null_builder: The buffer to append the null value to/// - original_view: The original view value/// - substr: The substring to append. Must be a valid substring of the original view/// - start_offset: The start offset of the substring in the view-pub fn make_and_append_view(+///+/// LLVM is apparently overly eager to inline this function into some hot loops,+/// which bloats them and regresses performance, so we disable inling for now.+#[inline(never)]+pub fn append_view(views_buffer: &mut Vec<u128>,- null_builder: &mut NullBufferBuilder,original_view: &u128,substr: &str,start_offset: u32,@@ -401,11 +405,9 @@ pub fn make_and_append_view(view.offset + start_offset,)} else {- // inline value does not need block id or offsetmake_view(substr.as_bytes(), 0, 0)};views_buffer.push(sub_view);- null_builder.append_non_null();}#[derive(Debug)]diff --git a/datafusion/functions/src/unicode/substr.rs b/datafusion/functions/src/unicode/substr.rsindex 43f7e58fa2..9e11534e2c 100644--- a/datafusion/functions/src/unicode/substr.rs+++ b/datafusion/functions/src/unicode/substr.rs@@ -17,13 +17,13 @@use std::sync::Arc;-use crate::strings::make_and_append_view;+use crate::strings::append_view;use crate::utils::make_scalar_function;use arrow::array::{- Array, ArrayRef, AsArray, Int64Array, NullBufferBuilder, StringArrayType,- StringViewArray, StringViewBuilder,+ Array, ArrayRef, AsArray, Int64Array, StringArrayType, StringViewArray,+ StringViewBuilder,};-use arrow::buffer::ScalarBuffer;+use arrow::buffer::{NullBuffer, ScalarBuffer};use arrow::datatypes::DataType;use datafusion_common::cast::as_int64_array;use datafusion_common::types::{@@ -278,15 +278,16 @@ fn string_view_substr(let enable_ascii_fast_path =enable_ascii_fast_path(&string_view_array, start_array, count_array_opt);+ // Combine null bitmaps from all inputs in bulk.+ let nulls = NullBuffer::union(+ NullBuffer::union(string_view_array.nulls(), start_array.nulls()).as_ref(),+ count_array_opt.and_then(|a| a.nulls()),+ );+let mut views_buf = Vec::with_capacity(string_view_array.len());- let mut null_builder = NullBufferBuilder::new(string_view_array.len());-- for i in 0..string_view_array.len() {- if string_view_array.is_null(i)- || start_array.is_null(i)- || count_array_opt.map(|a| a.is_null(i)).unwrap_or(false)- {- null_builder.append_null();++ for (i, raw_view) in string_view_array.views().iter().enumerate() {+ if nulls.as_ref().is_some_and(|n| n.is_null(i)) {views_buf.push(0);continue;}@@ -294,23 +295,15 @@ fn string_view_substr(let string = string_view_array.value(i);let start = start_array.value(i);let count = count_array_opt.map(|a| a.value(i));- let raw_view = string_view_array.views()[i];let (start, end) =get_true_start_end(string, start, count, enable_ascii_fast_path)?;let substr = &string[start..end];- make_and_append_view(- &mut views_buf,- &mut null_builder,- &raw_view,- substr,- start as u32,- );+ append_view(&mut views_buf, raw_view, substr, start as u32);}let views_buf = ScalarBuffer::from(views_buf);- let nulls_buf = null_builder.finish();// Safety:// (1) The blocks of the given views are all provided@@ -320,7 +313,7 @@ fn string_view_substr(let array = StringViewArray::new_unchecked(views_buf,string_view_array.data_buffers().to_vec(),- nulls_buf,+ nulls,);Ok(Arc::new(array) as ArrayRef)}@@ -336,13 +329,16 @@ wherelet enable_ascii_fast_path =enable_ascii_fast_path(&string_array, start_array, count_array_opt);+ // Combine null bitmaps from all inputs in bulk.+ let nulls = NullBuffer::union(+ NullBuffer::union(string_array.nulls(), start_array.nulls()).as_ref(),+ count_array_opt.and_then(|a| a.nulls()),+ );+let mut result_builder = StringViewBuilder::new();for i in 0..string_array.len() {- if string_array.is_null(i)- || start_array.is_null(i)- || count_array_opt.map(|a| a.is_null(i)).unwrap_or(false)- {+ if nulls.as_ref().is_some_and(|n| n.is_null(i)) {result_builder.append_null();continue;}