Apache DataFusion SQL Query Engine
perf: preallocate memory in `pad` (#23586)
theirix2e785088018b
diff --git a/datafusion/functions/src/unicode/common.rs b/datafusion/functions/src/unicode/common.rsindex 5b7262e29d..9f91e4f1b0 100644--- a/datafusion/functions/src/unicode/common.rs+++ b/datafusion/functions/src/unicode/common.rs@@ -50,6 +50,16 @@ pub(crate) fn try_as_scalar_i64(cv: &ColumnarValue) -> Option<i64> {}}+/// Estimates data capacity for `pad` based on `length_array` with row length.+/// For ASCII, one row is at most `target_len` bytes.+/// For UTF8, it could be larger+pub(crate) fn pad_data_capacity(length_array: &Int64Array) -> usize {+ length_array+ .iter()+ .flatten()+ .fold(0, |acc, len| acc.saturating_add(len as usize))+}+/// A trait for `left` and `right` byte slicing operationspub(crate) trait LeftRightSlicer {fn slice(string: &str, n: i64) -> Range<usize>;diff --git a/datafusion/functions/src/unicode/lpad.rs b/datafusion/functions/src/unicode/lpad.rsindex d27bc8633e..40bffeecf4 100644--- a/datafusion/functions/src/unicode/lpad.rs+++ b/datafusion/functions/src/unicode/lpad.rs@@ -178,7 +178,8 @@ impl ScalarUDFImpl for LPadFunc {}use super::common::{- StringCharLen, char_count_or_boundary, try_as_scalar_i64, try_as_scalar_str,+ StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,+ try_as_scalar_str,};/// Optimized lpad for constant target_len and fill arguments.@@ -373,7 +374,10 @@ whereT: OffsetSizeTrait,{let array = if let Some(fill_array) = fill_array {- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(+ string_array.len(),+ pad_data_capacity(length_array),+ );let mut fill_chars_buf = Vec::new();for ((string, target_len), fill) in string_array@@ -449,7 +453,10 @@ wherebuilder.finish()} else {- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(+ string_array.len(),+ pad_data_capacity(length_array),+ );for (string, target_len) in string_array.iter().zip(length_array.iter()) {if let (Some(string), Some(target_len)) = (string, target_len) {diff --git a/datafusion/functions/src/unicode/rpad.rs b/datafusion/functions/src/unicode/rpad.rsindex b3e14f9352..784a2037cf 100644--- a/datafusion/functions/src/unicode/rpad.rs+++ b/datafusion/functions/src/unicode/rpad.rs@@ -178,7 +178,8 @@ impl ScalarUDFImpl for RPadFunc {}use super::common::{- StringCharLen, char_count_or_boundary, try_as_scalar_i64, try_as_scalar_str,+ StringCharLen, char_count_or_boundary, pad_data_capacity, try_as_scalar_i64,+ try_as_scalar_str,};/// Optimized rpad for constant target_len and fill arguments.@@ -372,7 +373,10 @@ whereT: OffsetSizeTrait,{let array = if let Some(fill_array) = fill_array {- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(+ string_array.len(),+ pad_data_capacity(length_array),+ );let mut fill_chars_buf = Vec::new();for ((string, target_len), fill) in string_array@@ -450,7 +454,10 @@ wherebuilder.finish()} else {- let mut builder: GenericStringBuilder<T> = GenericStringBuilder::new();+ let mut builder: GenericStringBuilder<T> = GenericStringBuilder::with_capacity(+ string_array.len(),+ pad_data_capacity(length_array),+ );for (string, target_len) in string_array.iter().zip(length_array.iter()) {if let (Some(string), Some(target_len)) = (string, target_len) {